Initial commit of lexer

go.mod

@@ -0,0 +1,10 @@
+module git.geekproject.eu/halfdan/glox
+
+go 1.17
+
+require (
+	golang.org/x/mod v0.5.1 // indirect
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect
+	golang.org/x/tools v0.1.8 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+)

go.sum

@@ -0,0 +1,8 @@
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
+golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

lox/lox.go

@@ -0,0 +1,33 @@
+package lox
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+func RunFile(filename string) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	lexer := newLexer(string(bytes))
+	lexer.lex()
+}
+
+func RunFormat(filename string) {
+
+}
+
+func RunPrompt() {
+	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Split(bufio.ScanLines)
+	fmt.Print("> ")
+
+	for scanner.Scan() {
+		text := scanner.Text()
+		fmt.Printf("Input was: %s\n", text)
+		fmt.Print("> ")
+	}
+}

lox/scanner.go

@@ -0,0 +1,140 @@
+package lox
+
+type Lexer struct {
+	start   int
+	current int
+	line    int
+	column  int
+	source  []rune
+	tokens  []Token
+}
+
+func (lexer Lexer) lex() {
+
+	for !lexer.eof() {
+		lexer.start = lexer.current
+		c := lexer.advance()
+
+		switch c {
+		case '(':
+			lexer.addToken(LeftParen)
+		case ')':
+			lexer.addToken(RightParen)
+		case '{':
+			lexer.addToken(LeftBrace)
+		case '}':
+			lexer.addToken(RightBrace)
+		case ',':
+			lexer.addToken(Comma)
+		case '.':
+			lexer.addToken(Dot)
+		case '-':
+			lexer.addToken(Minus)
+		case '+':
+			lexer.addToken(Plus)
+		case ';':
+			lexer.addToken(Semicolon)
+		case '*':
+			lexer.addToken(Star)
+		case '!':
+			if lexer.match('=') {
+				lexer.addToken(BangEqual)
+			} else {
+				lexer.addToken(Bang)
+			}
+		case '=':
+			if lexer.match('=') {
+				lexer.addToken(EqualEqual)
+			} else {
+				lexer.addToken(Equal)
+			}
+		case '<':
+			if lexer.match('=') {
+				lexer.addToken(LessEqual)
+			} else {
+				lexer.addToken(Less)
+			}
+		case '>':
+			if lexer.match('=') {
+				lexer.addToken(GreaterEqual)
+			} else {
+				lexer.addToken(Greater)
+			}
+		case '/':
+			if lexer.match('/') {
+				for lexer.peek() != '\n' && !lexer.eof() {
+					lexer.advance()
+				}
+			} else {
+				lexer.addToken(Slash)
+			}
+		case ' ', '\r', '\t': // Ignore whitespace
+			continue
+		case '\n':
+			lexer.line++
+			lexer.column = 0
+		case '"':
+			for lexer.peek() != '"' && !lexer.eof() {
+				if lexer.peek() == '\n' {
+					lexer.line++
+					lexer.column--
+				}
+				lexer.advance()
+			}
+
+			// if lexer.eof() {
+			// 	// Add error (unterminated string)
+			// }
+
+			// Consume the terminating "
+			lexer.advance()
+
+			//value := lexer.source[lexer.start+1 : lexer.current-1]
+			//lexer.addToken(String, value)
+		}
+	}
+	lexer.addToken(Eof)
+}
+
+func (s *Lexer) addToken(tt TokenType) {
+	s.tokens = append(s.tokens, *makeToken(tt, "", nil, s.line, s.column))
+}
+
+// eof returns true if the read pointer has reached the end of the source file
+func (s Lexer) eof() bool {
+	return s.current >= len(s.source)
+}
+
+// peek looks one rune ahead without advancing the cursor
+func (s Lexer) peek() rune {
+	if s.eof() {
+		return '\u0004' // emit end of file character
+	}
+	return s.source[s.current]
+}
+
+// advance increments the read pointer and returns the rune at that position
+func (s *Lexer) advance() rune {
+	r := s.source[s.current]
+	s.current++
+	s.column++
+	return r
+}
+
+// match tried to match a single rune and advances the read pointer if it does.
+func (s *Lexer) match(expected rune) bool {
+	if s.eof() {
+		return false
+	}
+	if s.source[s.current] != expected {
+		return false
+	}
+
+	s.current++
+	s.column++
+	return true
+}
+
+func newLexer(source string) *Lexer {
+	return &Lexer{line: 1, column: 0, source: []rune(source)}
+}

lox/token.go

@@ -0,0 +1,72 @@
+package lox
+
+import "fmt"
+
+//go:generate stringer -type=TokenType
+type TokenType int
+
+const (
+	// Single character tokens
+	LeftParen TokenType = iota
+	RightParen
+	LeftBrace
+	RightBrace
+	Comma
+	Dot
+	Minus
+	Plus
+	Semicolon
+	Slash
+	Star
+
+	// One or two character tokens
+	Bang
+	BangEqual
+	Equal
+	EqualEqual
+	Greater
+	GreaterEqual
+	Less
+	LessEqual
+
+	// Literals
+	Identifier
+	String
+	Number
+
+	// Keywords
+	And
+	Class
+	Else
+	False
+	Fun
+	For
+	If
+	Nil
+	Or
+	Print
+	Return
+	Super
+	This
+	True
+	Var
+	While
+
+	Eof
+)
+
+type Token struct {
+	tokenType TokenType
+	lexeme    string
+	literal   interface{}
+	line      int
+	column    int
+}
+
+func (t Token) String() string {
+	return fmt.Sprintf("%v", t)
+}
+
+func makeToken(tt TokenType, lexeme string, lit interface{}, line, column int) *Token {
+	return &Token{tokenType: tt, lexeme: lexeme, literal: lit, line: line, column: column}
+}

lox/tokentype_string.go

@@ -0,0 +1,61 @@
+// Code generated by "stringer -type=TokenType"; DO NOT EDIT.
+
+package lox
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[LeftParen-0]
+	_ = x[RightParen-1]
+	_ = x[LeftBrace-2]
+	_ = x[RightBrace-3]
+	_ = x[Comma-4]
+	_ = x[Dot-5]
+	_ = x[Minus-6]
+	_ = x[Plus-7]
+	_ = x[Semicolon-8]
+	_ = x[Slash-9]
+	_ = x[Star-10]
+	_ = x[Bang-11]
+	_ = x[BangEqual-12]
+	_ = x[Equal-13]
+	_ = x[EqualEqual-14]
+	_ = x[Greater-15]
+	_ = x[GreaterEqual-16]
+	_ = x[Less-17]
+	_ = x[LessEqual-18]
+	_ = x[Identifier-19]
+	_ = x[String-20]
+	_ = x[Number-21]
+	_ = x[And-22]
+	_ = x[Class-23]
+	_ = x[Else-24]
+	_ = x[False-25]
+	_ = x[Fun-26]
+	_ = x[For-27]
+	_ = x[If-28]
+	_ = x[Nil-29]
+	_ = x[Or-30]
+	_ = x[Print-31]
+	_ = x[Return-32]
+	_ = x[Super-33]
+	_ = x[This-34]
+	_ = x[True-35]
+	_ = x[Var-36]
+	_ = x[While-37]
+	_ = x[Eof-38]
+}
+
+const _TokenType_name = "LeftParenRightParenLeftBraceRightBraceCommaDotMinusPlusSemicolonSlashStarBangBangEqualEqualEqualEqualGreaterGreaterEqualLessLessEqualIdentifierStringNumberAndClassElseFalseFunForIfNilOrPrintReturnSuperThisTrueVarWhileEof"
+
+var _TokenType_index = [...]uint8{0, 9, 19, 28, 38, 43, 46, 51, 55, 64, 69, 73, 77, 86, 91, 101, 108, 120, 124, 133, 143, 149, 155, 158, 163, 167, 172, 175, 178, 180, 183, 185, 190, 196, 201, 205, 209, 212, 217, 220}
+
+func (i TokenType) String() string {
+	if i < 0 || i >= TokenType(len(_TokenType_index)-1) {
+		return "TokenType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _TokenType_name[_TokenType_index[i]:_TokenType_index[i+1]]
+}

main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	lox "git.geekproject.eu/halfdan/glox/lox"
+)
+
+// glox:
+//	run		- run a scripts
+//	build	- build a binary
+//	fmt		- format file
+//	sh 		- run interactive shell
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+	}
+
+	switch os.Args[1] {
+	case "run":
+		if len(os.Args) < 3 {
+			fmt.Print("glox run: no lox file listed\n")
+			os.Exit(2)
+		}
+		lox.RunFile(os.Args[2])
+	case "fmt":
+		if len(os.Args) < 3 {
+			fmt.Print("glox fmt: no lox file listed\n")
+			os.Exit(2)
+		}
+		lox.RunFormat(os.Args[2])
+	case "sh":
+		lox.RunPrompt()
+	default:
+		usage()
+	}
+}
+
+func usage() {
+	fmt.Print("glox is a tool for running Lox code.\n\n")
+	fmt.Print("Usage:\n\n")
+	fmt.Print("\tglox <command> [arguments]\n\n")
+	fmt.Print("The commands are:\n\n")
+	fmt.Print("\trun\trun a lox script\n")
+	fmt.Print("\tfmt\tformat input file\n")
+	fmt.Print("\tsh\tstart interactive shell\n\n")
+	os.Exit(2)
+}