Initial commit of lexer

lox/token.go

@@ -0,0 +1,72 @@
+package lox
+
+import "fmt"
+
+//go:generate stringer -type=TokenType
+type TokenType int
+
+const (
+	// Single character tokens
+	LeftParen TokenType = iota
+	RightParen
+	LeftBrace
+	RightBrace
+	Comma
+	Dot
+	Minus
+	Plus
+	Semicolon
+	Slash
+	Star
+
+	// One or two character tokens
+	Bang
+	BangEqual
+	Equal
+	EqualEqual
+	Greater
+	GreaterEqual
+	Less
+	LessEqual
+
+	// Literals
+	Identifier
+	String
+	Number
+
+	// Keywords
+	And
+	Class
+	Else
+	False
+	Fun
+	For
+	If
+	Nil
+	Or
+	Print
+	Return
+	Super
+	This
+	True
+	Var
+	While
+
+	Eof
+)
+
+type Token struct {
+	tokenType TokenType
+	lexeme    string
+	literal   interface{}
+	line      int
+	column    int
+}
+
+func (t Token) String() string {
+	return fmt.Sprintf("%v", t)
+}
+
+func makeToken(tt TokenType, lexeme string, lit interface{}, line, column int) *Token {
+	return &Token{tokenType: tt, lexeme: lexeme, literal: lit, line: line, column: column}
+}