input

Documentation

Overview

Package input implements a two-buffer input reader.

This package is particularly suited for implementing lexical analyzers and parsers, offering robust support for managing lexemes, handling multi-byte UTF-8 characters, and processing character streams.

The two-buffer technique handles large input streams by splitting the buffer into two halves that are alternately reloaded, ensuring efficient processing without frequent I/O operations.

For more information and details, see "Compilers: Principles, Techniques, and Tools (2nd Edition)".

Index

• type Input
  • func New(filename string, src io.Reader, n int) (*Input, error)
  • func (i *Input) Lexeme() (string, lexer.Position)
  • func (i *Input) Next() (rune, error)
  • func (i *Input) Retract()
  • func (i *Input) Skip() lexer.Position
• type InputError
  • func (e *InputError) Error() string

Examples

• Input
• Input.Lexeme
• Input.Next
• Input.Retract
• Input.Skip

Types

type Input

type Input struct {
	// contains filtered or unexported fields
}

Input implements the two-buffer scheme for reading the input characters.

For more information and details, see "Compilers: Principles, Techniques, and Tools (2nd Edition)".

Example

package main

import (
	"fmt"
	"os"

	"github.com/moorara/algo/lexer/input"
)

func main() {
	bufferSize := 4096

	file, err := os.Open("./fixture/lorem_ipsum")
	if err != nil {
		panic(err)
	}

	in, err := input.New("lorem_ipsum", file, bufferSize)
	if err != nil {
		panic(err)
	}

	// advanceDFA simulates a determinist finite automata for identifying words.
	advanceDFA := func(state int, r rune) int {
		switch state {
		case 0:
			switch r {
			case ' ', ',', '.', '\n':
				return 0
			default:
				return 1
			}

		case 1:
			switch r {
			case ' ', ',', '.', '\n':
				return 3
			default:
				return 2
			}

		case 2:
			switch r {
			case ' ', ',', '.', '\n':
				return 3
			default:
				return 2
			}

		case 3:
			switch r {
			case ' ', ',', '.', '\n':
				return 0
			default:
				return 1
			}

		default:
			return -1
		}
	}

	// Reads runes from the input and feeds them into the DFA.
	var state int
	var r rune
	for r, err = in.Next(); err == nil; r, err = in.Next() {
		state = advanceDFA(state, r)
		switch state {
		case 1:
			in.Retract()
			in.Skip()
		case 3:
			in.Retract()
			lexeme, pos := in.Lexeme()
			fmt.Printf("Lexeme %q at %s\n", lexeme, pos)
		}
	}

	fmt.Println(err)
}

func New

func New(filename string, src io.Reader, n int) (*Input, error)

New creates a new input buffer of size N. N usually should be the size of a disk block.

func (*Input) Lexeme

func (i *Input) Lexeme() (string, lexer.Position)

Lexeme returns the current lexeme alongside its position.

Example

package main

import (
	"fmt"
	"strings"

	"github.com/moorara/algo/lexer/input"
)

func main() {
	bufferSize := 4096
	src := strings.NewReader(`Lorem ipsum dolor sit amet, consectetur adipiscing elit,
		sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)

	in, err := input.New("lorem_ipsum", src, bufferSize)
	if err != nil {
		panic(err)
	}

	for range 5 {
		r, err := in.Next()
		if err != nil {
			panic(err)
		}

		fmt.Printf("rune: %c\n", r)
	}

	// Reading the current lexeme.
	lexeme, pos := in.Lexeme()
	fmt.Printf("lexeme: %q  position: %s\n", lexeme, pos)
}

func (*Input) Next

func (i *Input) Next() (rune, error)

Next advances to the next rune in the input and returns it. If the end of the input is reached, it returns the io.EOF error.

Example

package main

import (
	"fmt"
	"strings"

	"github.com/moorara/algo/lexer/input"
)

func main() {
	bufferSize := 4096
	src := strings.NewReader(`Lorem ipsum dolor sit amet, consectetur adipiscing elit,
		sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)

	in, err := input.New("lorem_ipsum", src, bufferSize)
	if err != nil {
		panic(err)
	}

	// Reading the next rune.
	r, err := in.Next()
	if err != nil {
		panic(err)
	}

	fmt.Printf("rune: %c\n", r)
}

func (*Input) Retract

func (i *Input) Retract()

Retract recedes to the last rune in the input.

Example

package main

import (
	"fmt"
	"strings"

	"github.com/moorara/algo/lexer/input"
)

func main() {
	bufferSize := 4096
	src := strings.NewReader(`Lorem ipsum dolor sit amet, consectetur adipiscing elit,
		sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)

	in, err := input.New("lorem_ipsum", src, bufferSize)
	if err != nil {
		panic(err)
	}

	r, err := in.Next()
	if err != nil {
		panic(err)
	}

	fmt.Printf("rune: %c\n", r)

	r, err = in.Next()
	if err != nil {
		panic(err)
	}

	fmt.Printf("rune: %c\n", r)

	// Undoing the last rune read.
	in.Retract()

	r, err = in.Next()
	if err != nil {
		panic(err)
	}

	fmt.Printf("rune: %c\n", r)
}

func (*Input) Skip

func (i *Input) Skip() lexer.Position

Skip skips over the pending lexeme in the input.

Example

package main

import (
	"fmt"
	"strings"

	"github.com/moorara/algo/lexer/input"
)

func main() {
	bufferSize := 4096
	src := strings.NewReader(`Lorem ipsum dolor sit amet, consectetur adipiscing elit,
		sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)

	in, err := input.New("lorem_ipsum", src, bufferSize)
	if err != nil {
		panic(err)
	}

	for range 6 {
		if _, err = in.Next(); err != nil {
			panic(err)
		}
	}

	// Skiping the current lexeme.
	pos := in.Skip()
	fmt.Printf("position of skipped lexeme: %s\n", pos)

	for range 5 {
		if _, err = in.Next(); err != nil {
			panic(err)
		}
	}

	// Reading the next lexeme.
	lexeme, pos := in.Lexeme()
	fmt.Printf("lexeme: %q  position: %s\n", lexeme, pos)
}

type InputError

type InputError struct {
	Description string
	Pos         lexer.Position
}

InputError represents an error encountered when reading from an input source.

func (*InputError) Error

func (e *InputError) Error() string

Error implements the error interface. It returns a formatted string describing the error in detail.