parsekit

// Let's write a small example for parsing a really basic calculator.
// The calculator understands input that looks like:
//
//	10 + 20 - 8+4
//
// So positive numbers that can be either added or substracted, and whitespace
// is ignored.
package main

import (
	"fmt"
	"strconv"

	"git.makaay.nl/mauricem/go-parsekit"
)

// When writing a parser, it's a good start to use the parser/combinator
// functionality of parsekit to create some TokenHandler functions. These functions
// can later be used in the parser state machine to check for matching strings
// on the input data.
//
// For the calculator, we only need a definition of "number, surrounded by
// optional whitespace". Skipping whitespace could be a part of the StateHandler
// functions below too, but including it in a TokenHandler makes things really
// practical.
func createNumberMatcher() parsekit.TokenHandler {
	// Easy access to parsekit definition.
	c, a, m := parsekit.C, parsekit.A, parsekit.M

	whitespace := m.Drop(c.Opt(a.Whitespace))
	return c.Seq(whitespace, c.OneOrMore(a.Digit), whitespace)
}

var calcNumber = createNumberMatcher()

// We need to define the ItemTypes that we will use for emitting Items
// during the parsing process.
const (
	numberType parsekit.ItemType = iota
	addType
	subtractType
)

// We also need to define the state machine for parsing the input.
// The state machine is built up from functions that match the StateHandler
// signature: func(*parsekit.ParseAPI)
// The ParseAPI struct holds the internal state for the parser and it provides
// some methods that form the API for your StateHandler implementation.

// State: expect a number. When a number is found on the input,
// it is accepted in the parser's string buffer, after which that buffer is
// emitted as a numberType item. Then we tell the state machine to continue
// with the calcWaitForOperatorOrEndOfInput state.
// When no number is found, the parser will emit an error, explaining that
// "a number" was expected.
func calcWaitForNumber(p *parsekit.ParseAPI) {
	p.Expects("a number")
	if p.On(calcNumber).Accept() {
		p.EmitLiteral(numberType)
		p.RouteTo(calcWaitForOperatorOrEndOfInput)
	}
}

// State: expect a plus or minus operator. When one of those is found, the
// appropriate Item is emitted and the parser is sent back to the
// numberHandler to find the next number on the input. When no operator is
// found, then the parser is told to expect the end of the input. When more
// input data are available (which are obviously wrong data since they do
// not match our syntax), the parser will emit an error.
func calcWaitForOperatorOrEndOfInput(p *parsekit.ParseAPI) {
	switch {
	case p.On(a.Plus).Accept():
		p.EmitLiteral(addType)
		p.RouteTo(calcWaitForNumber)
	case p.On(a.Minus).Accept():
		p.EmitLiteral(subtractType)
		p.RouteTo(calcWaitForNumber)
	default:
		p.ExpectEndOfFile()
	}
}

// All is ready for our parser. We now can create a new Parser struct.
// We need to tell it what StateHandler to start with. In our case, it is the
// calcWaitForNumber state, since the calculation must start with a number.
var calcParser = parsekit.NewParser(calcWaitForNumber)

func main() {
	// Let's feed the parser some input to work with. This provides us with
	// a parse run for that input.
	run := calcParser.Parse(" 153+22 + 31-4 -\t 6+42 ")

	// We can now step through the results of the parsing process by repeated
	// calls to run.Next(). Next() returns either the next parse item, a parse
	// error or an end of file. Let's dump the parse results and handle the
	// computation while we're at it.
	// TODO this in convoluted for people using the parser code I think. Maybe use three output data types instead?
	sum := 0
	op := +1
	for {
		item, err, ok := run.Next()
		switch {
		case !ok && err == nil:
			fmt.Println("End of file reached")
			fmt.Println("Outcome of computation:", sum)
			return
		case !ok:
			fmt.Printf("Error: %s\n", err)
			return
		default:
			fmt.Printf("Type: %d, Value: %q\n", item.Type, item.Value)
			switch {
			case item.Type == addType:
				op = +1
			case item.Type == subtractType:
				op = -1
			case item.Type == numberType:
				nr, err := strconv.Atoi(item.Value)
				if err != nil {
					fmt.Printf("Error: invalid number %s: %s\n", item.Value, err)
					return
				}
				sum += op * nr
			}
		}
	}

}

Output:

Type: 0, Value: "153"
Type: 1, Value: "+"
Type: 0, Value: "22"
Type: 1, Value: "+"
Type: 0, Value: "31"
Type: 2, Value: "-"
Type: 0, Value: "4"
Type: 2, Value: "-"
Type: 0, Value: "6"
Type: 1, Value: "+"
Type: 0, Value: "42"
End of file reached
Outcome of computation: 238

// In this example, a Parser is created which can parse and normalize Dutch postcodes
// The implementation uses only TokenHandler functions and does not implement a
// full-fledged state-based Parser for it.
package main

import (
	"fmt"

	"git.makaay.nl/mauricem/go-parsekit"
)

func createPostcodeMatcher() *parsekit.Matcher {
	// Easy access to the parsekit definitions.
	c, a, m := parsekit.C, parsekit.A, parsekit.M

	// TokenHandler functions are created and combined to satisfy these rules:
	// - A Dutch postcode consists of 4 digits and 2 letters (1234XX).
	// - The first digit is never a zero.
	// - A space between letters and digits is optional.
	// - It is good form to write the letters in upper case.
	// - It is good form to use a single space between digits and letters.
	digitNotZero := c.Except(c.Rune('0'), a.Digit)
	pcDigits := c.Seq(digitNotZero, c.Rep(3, a.Digit))
	pcLetter := c.Any(a.ASCIILower, a.ASCIIUpper)
	pcLetters := m.ToUpper(c.Seq(pcLetter, pcLetter))
	space := m.Replace(c.Opt(a.Whitespace), " ")
	postcode := c.Seq(pcDigits, space, pcLetters)

	// Create a Matcher, which wraps the 'postcode' TokenHandler and allows
	// us to match some input against that handler.
	return parsekit.NewMatcher(postcode, "a Dutch postcode")
}

func main() {
	pcParser := createPostcodeMatcher()

	for i, input := range []string{
		"1234  AB",
		"2233Ab",
		"1001\t\tab",
		"1818ab",
		"1234",
		"huh",
	} {
		output, err, ok := pcParser.Parse(input)
		if !ok {
			fmt.Printf("[%d] Input: %q Error: %s\n", i, input, err)
		} else {
			fmt.Printf("[%d] Input: %q Output: %s\n", i, input, output)
		}
	}
}

Output:

[0] Input: "1234  AB" Output: 1234 AB
[1] Input: "2233Ab" Output: 2233 AB
[2] Input: "1001\t\tab" Output: 1001 AB
[3] Input: "1818ab" Output: 1818 AB
[4] Input: "1234" Error: unexpected character '1' (expected a Dutch postcode)
[5] Input: "huh" Error: unexpected character 'h' (expected a Dutch postcode)

// In this example, a parser is created that is able to parse input that looks
// like "Hello, <name>!", and that extracts the name from it.
//
// The implementation uses only parser/combinator TokenHandler functions and does
// not implement a full-fledged state-based Parser for it. If you want to see the
// same kind of functionality, implementated using a Paser, take a look at the
// HelloWorldUsingParser example.
package main

import (
	"fmt"

	"git.makaay.nl/mauricem/go-parsekit"
)

func createHelloMatcher() *parsekit.Matcher {
	// Easy access to parsekit definition.
	c, a, m := parsekit.C, parsekit.A, parsekit.M

	// Using the parser/combinator support of parsekit, we create a TokenHandler function
	// that does all the work. The 'greeting' TokenHandler matches the whole input and
	// drops all but the name from it.
	hello := c.StrNoCase("hello")
	comma := c.Seq(c.Opt(a.Whitespace), a.Comma, c.Opt(a.Whitespace))
	separator := c.Any(comma, a.Whitespace)
	name := c.OneOrMore(c.Not(a.Excl))
	greeting := c.Seq(m.Drop(hello), m.Drop(separator), name, m.Drop(a.Excl))

	// Create a Matcher, which wraps the 'greeting' TokenHandler and allows
	// us to match some input against that handler.
	return parsekit.NewMatcher(greeting, "a friendly greeting")
}

func main() {
	parser := createHelloMatcher()

	for i, input := range []string{
		"Hello, world!",
		"HELLO ,Johnny!",
		"hello , Bob123!",
		"hello Pizza!",
		"Oh no!",
		"Hello, world",
	} {
		output, err, ok := parser.Parse(input)
		if !ok {
			fmt.Printf("[%d] Input: %q Error: %s\n", i, input, err)
		} else {
			fmt.Printf("[%d] Input: %q Output: %s\n", i, input, output)
		}
	}
}

Output:

[0] Input: "Hello, world!" Output: world
[1] Input: "HELLO ,Johnny!" Output: Johnny
[2] Input: "hello , Bob123!" Output: Bob123
[3] Input: "hello Pizza!" Output: Pizza
[4] Input: "Oh no!" Error: unexpected character 'O' (expected a friendly greeting)
[5] Input: "Hello, world" Error: unexpected character 'H' (expected a friendly greeting)

// In this example, a parser is created that is able to parse input that looks
// like "Hello, <name>!", and that extracts the name from it.
//
// This implementation uses a state-based Parser for it, and it does not implement
// any custom parser/combinator TokenHandler functions. Note that things are much
// easier to implement using custom TokenHandlers (see the other HelloWorldUsingMatcher
// example for this). Doing this fully parser-based implementation is mainly for your
// learning pleasure.
//
// One big difference between the Matcher-based example and this one, is that the
// state-based parser reports errors much more fine-grained. This might or might
// not be useful for your specific use case.
package main

import (
	"fmt"
	"strings"

	"git.makaay.nl/mauricem/go-parsekit"
)

const greeteeItem parsekit.ItemType = 1

func stateStartOfGreeting(p *parsekit.ParseAPI) {
	c := parsekit.C
	p.Expects("hello")
	if p.On(c.StrNoCase("hello")).Skip() {
		p.RouteTo(stateComma)
	}
}

func stateComma(p *parsekit.ParseAPI) {
	a := parsekit.A
	p.Expects("comma")
	switch {
	case p.On(a.Whitespace).Skip():
		p.RouteRepeat()
	case p.On(a.Comma).Skip():
		p.RouteTo(stateName)
	}
}

func stateName(p *parsekit.ParseAPI) {
	a := parsekit.A
	p.Expects("name")
	switch {
	case p.On(a.Excl).Skip():
		p.RouteTo(stateEndOfGreeting)
	case p.On(a.AnyRune).Accept():
		p.RouteRepeat()
	}
}

func stateEndOfGreeting(p *parsekit.ParseAPI) {
	p.Expects("end of greeting")
	if p.On(a.EndOfFile).Stay() {
		name := strings.TrimSpace(p.BufLiteral())
		if name == "" {
			p.EmitError("The name cannot be empty")
		} else {
			p.Emit(greeteeItem, name)
		}
	}
}

func createHelloParser() *parsekit.Parser {
	return parsekit.NewParser(stateStartOfGreeting)
}

func main() {
	parser := createHelloParser()

	for i, input := range []string{
		"Hello, world!",
		"HELLO ,Johnny!",
		"hello , Bob123!",
		"hello Pizza!",
		"",
		" ",
		"hello",
		"hello,",
		"hello , ",
		"hello , Droopy",
		"hello , Droopy!",
		"hello , \t \t Droopy \t !",
		"Oh no!",
		"hello,!",
	} {
		item, err, ok := parser.Parse(input).Next()
		if !ok {
			fmt.Printf("[%d] Input: %q Error: %s\n", i, input, err)
		} else {
			fmt.Printf("[%d] Input: %q Output: %s\n", i, input, item.Value)
		}
	}
}

Output:

[0] Input: "Hello, world!" Output: world
[1] Input: "HELLO ,Johnny!" Output: Johnny
[2] Input: "hello , Bob123!" Output: Bob123
[3] Input: "hello Pizza!" Error: unexpected character 'P' (expected comma)
[4] Input: "" Error: unexpected end of file (expected hello)
[5] Input: " " Error: unexpected character ' ' (expected hello)
[6] Input: "hello" Error: unexpected end of file (expected comma)
[7] Input: "hello," Error: unexpected end of file (expected name)
[8] Input: "hello , " Error: unexpected end of file (expected name)
[9] Input: "hello , Droopy" Error: unexpected end of file (expected name)
[10] Input: "hello , Droopy!" Output: Droopy
[11] Input: "hello , \t \t Droopy \t !" Output: Droopy
[12] Input: "Oh no!" Error: unexpected character 'O' (expected hello)
[13] Input: "hello,!" Error: The name cannot be empty