lex

type KeywordTokenizer struct {
	Keyword string
	Token   TokenType
}

type PrefixedTokenizer struct {
	Prefix string
	Token  TokenType
}

type SpecificTokenizer interface {
	Tokenize(txt *core.SourceView) (Token, error)
}

type Token struct {
	Type TokenType
	View core.UnmanagedSourceView
}

type TokenType uint8

type Tokenizer interface {
	Tokenize(core.SourceView) ([]Token, error)
}

type WordTokenizer struct {
	Token TokenType
}