Initial commit

1 files changed, 172 insertions, 0 deletions

diff --git a/token.go b/token.go
new file mode 100644
index 0000000..a33a554
--- /dev/null
+++ b/token.go
@@ -0,0 +1,172 @@
+package main
+
+type tokenKind int
+
+func (k tokenKind) String() string {
+	switch k {
+	case plusEquals:
+		return "+="
+	case plus:
+		return "+"
+	case minusEquals:
+		return "-="
+	case minus:
+		return "-"
+	case starEquals:
+		return "*="
+	case star:
+		return "*"
+	case slashEquals:
+		return "/="
+	case slash:
+		return "/"
+	case lparen:
+		return "("
+	case rparen:
+		return ")"
+	case lbracket:
+		return "["
+	case rbracket:
+		return "]"
+	case lbrace:
+		return "{"
+	case rbrace:
+		return "}"
+	case comma:
+		return ","
+	case dot:
+		return "."
+	case colon:
+		return ":"
+	case semicolon:
+		return ";"
+	case doubleEquals:
+		return "=="
+	case equals:
+		return "="
+	case bangEquals:
+		return "!="
+	case bang:
+		return "!"
+	case tildeEquals:
+		return "~="
+	case tilde:
+		return "~"
+	case caretEquals:
+		return "^="
+	case caret:
+		return "^"
+	case doubleAmpersand:
+		return "&&"
+	case ampersandEquals:
+		return "&="
+	case ampersand:
+		return "&"
+	case doublePipe:
+		return "||"
+	case pipeEquals:
+		return "|="
+	case pipe:
+		return "|"
+	case doubleLangleEquals:
+		return "<<="
+	case doubleLangle:
+		return "<<"
+	case langleEquals:
+		return "<="
+	case langle:
+		return "<"
+	case doubleRangleEquals:
+		return ">>="
+	case doubleRangle:
+		return ">>"
+	case rangleEquals:
+		return ">="
+	case rangle:
+		return ">"
+	case identifier:
+		return "identifier"
+	case str:
+		return "string"
+	case number:
+		return "number"
+	case export:
+		return "export"
+	case function:
+		return "func"
+	case ret:
+		return "return"
+	}
+
+	return "invalid token"
+}
+
+const (
+	none tokenKind = iota
+	plusEquals
+	plus
+	minusEquals
+	minus
+	starEquals
+	star
+	slashEquals
+	slash
+	lparen
+	rparen
+	lbracket
+	rbracket
+	lbrace
+	rbrace
+	comma
+	dot
+	colon
+	semicolon
+	doubleEquals
+	equals
+	bangEquals
+	bang
+	tildeEquals
+	tilde
+	caretEquals
+	caret
+	doubleAmpersand
+	ampersandEquals
+	ampersand
+	doublePipe
+	pipeEquals
+	pipe
+	doubleLangleEquals
+	doubleLangle
+	langleEquals
+	langle
+	doubleRangleEquals
+	doubleRangle
+	rangleEquals
+	rangle
+	identifier
+	str
+	number
+	export
+	function
+	ret
+)
+
+type token struct {
+	kind  tokenKind
+	value string
+	line  int
+}
+
+func (t token) String() string {
+	if t.value != "" {
+		return t.kind.String() + " " + t.value
+	} else {
+		return t.kind.String()
+	}
+}
+
+type tokens struct {
+	src   <-chan token
+	toks  []token
+	index int
+}