Add extra tests and line counting

1 files changed, 265 insertions, 0 deletions

diff --git a/lexer_test.go b/lexer_test.go
new file mode 100644
index 0000000..25da17f
--- /dev/null
+++ b/lexer_test.go
@@ -0,0 +1,265 @@
+package lexer
+
+import (
+	"fmt"
+	"testing"
+)
+
+const (
+	NumberToken TokenType = iota
+	OpToken
+	IdentToken
+)
+
+func NumberState(l *Lexer) StateFunc {
+	l.AcceptRun("0123456789")
+	l.Emit(NumberToken)
+	if l.Peek() == '.' {
+		l.Next()
+		l.Emit(OpToken)
+		return IdentState
+	}
+
+	return nil
+}
+
+func IdentState(l *Lexer) StateFunc {
+	r := l.Next()
+	for (r >= 'a' && r <= 'z') || r == '_' {
+		r = l.Next()
+	}
+	l.Backup()
+	l.Emit(IdentToken)
+
+	return WhitespaceState
+}
+
+func NewlineState(l *Lexer) StateFunc {
+	l.AcceptRun("0123456789")
+	l.Emit(NumberToken)
+	l.SkipWhitespace()
+	l.Ignore()
+	l.AcceptRun("0123456789")
+	l.Emit(NumberToken)
+	l.SkipWhitespace()
+
+	return nil
+}
+
+func WhitespaceState(l *Lexer) StateFunc {
+	r := l.Next()
+	if r == EOFRune {
+		return nil
+	}
+
+	if r != ' ' && r != '\t' && r != '\n' && r != '\r' {
+		l.Error(fmt.Sprintf("unexpected token %q", r))
+		return nil
+	}
+
+	l.Accept(" \t\n\r")
+	l.Ignore()
+
+	return NumberState
+}
+
+func TestMovingThroughString(t *testing.T) {
+	l := New("123", nil)
+	run := []struct {
+		s string
+		r rune
+	}{
+		{"1", '1'},
+		{"12", '2'},
+		{"123", '3'},
+		{"123", EOFRune},
+	}
+
+	for _, test := range run {
+		r := l.Next()
+		if r != test.r {
+			t.Errorf("Expected %q but got %q", test.r, r)
+			return
+		}
+
+		if l.Current() != test.s {
+			t.Errorf("Expected %q but got %q", test.s, l.Current())
+			return
+		}
+	}
+}
+
+func TestNumbers(t *testing.T) {
+	l := New("123", NumberState)
+	l.Start()
+	tok, done := l.NextToken()
+	if done {
+		t.Error("Expected a token, but lexer was finished")
+		return
+	}
+
+	if tok.Type != NumberToken {
+		t.Errorf("Expected a %v but got %v", NumberToken, tok.Type)
+		return
+	}
+
+	if tok.Value != "123" {
+		t.Errorf("Expected %q but got %q", "123", tok.Value)
+		return
+	}
+
+	tok, done = l.NextToken()
+	if !done {
+		t.Error("Expected the lexer to be done, but it wasn't.")
+		return
+	}
+
+	if tok != nil {
+		t.Errorf("Expected a nil token, but got %v", *tok)
+		return
+	}
+}
+
+func TestNewlines(t *testing.T) {
+	src := `123
+456
+789`
+	l := New(src, NewlineState)
+	l.Start()
+	tok, done := l.NextToken()
+	if done {
+		t.Error("Expected the lexer to not be done, but it was.")
+		return
+	}
+
+	if tok.Type != NumberToken {
+		t.Errorf("Expected a number token but got %v", *tok)
+		return
+	}
+
+	if tok.Value != "123" {
+		t.Errorf("Expected 123 but got %q", tok.Value)
+		return
+	}
+
+	if tok.Line != 1 {
+		t.Fatalf("Expected line 1 but got %d", tok.Line)
+	}
+
+	tok, done = l.NextToken()
+	if done {
+		t.Error("Expected the lexer to not be done, but it was.")
+		return
+	}
+
+	if tok.Type != NumberToken {
+		t.Errorf("Expected a number token but got %v", *tok)
+		return
+	}
+
+	if tok.Value != "456" {
+		t.Errorf("Expected 456 but got %q", tok.Value)
+		return
+	}
+
+	if tok.Line != 2 {
+		t.Fatalf("Expected line 2 but got %d", tok.Line)
+	}
+}
+
+func TestBackup(t *testing.T) {
+	l := New("1", nil)
+	r := l.Next()
+	if r != '1' {
+		t.Errorf("Expected %q but got %q", '1', r)
+		return
+	}
+
+	if l.Current() != "1" {
+		t.Errorf("Expected %q but got %q", "1", l.Current())
+		return
+	}
+
+	l.Backup()
+	if l.Current() != "" {
+		t.Errorf("Expected empty string, but got %q", l.Current())
+		return
+	}
+}
+
+func TestWhitespace(t *testing.T) {
+	l := New("    1", NumberState)
+	l.Start()
+	l.SkipWhitespace()
+
+	tok, done := l.NextToken()
+	if done {
+		t.Fatal("Expected token to be !done, but it was.")
+	}
+
+	if tok.Type != NumberToken {
+		t.Fatalf("Expected number token, but got %v", *tok)
+	}
+}
+
+func TestMultipleTokens(t *testing.T) {
+	cases := []struct {
+		tokType TokenType
+		val     string
+	}{
+		{NumberToken, "123"},
+		{OpToken, "."},
+		{IdentToken, "hello"},
+		{NumberToken, "675"},
+		{OpToken, "."},
+		{IdentToken, "world"},
+	}
+
+	l := New("123.hello  675.world", NumberState)
+	l.Start()
+
+	for _, c := range cases {
+		tok, done := l.NextToken()
+		if done {
+			t.Error("Expected there to be more tokens, but there weren't")
+			return
+		}
+
+		if c.tokType != tok.Type {
+			t.Errorf("Expected token type %v but got %v", c.tokType, tok.Type)
+			return
+		}
+
+		if c.val != tok.Value {
+			t.Errorf("Expected %q but got %q", c.val, tok.Value)
+			return
+		}
+	}
+
+	tok, done := l.NextToken()
+	if !done {
+		t.Error("Expected the lexer to be done, but it wasn't.")
+		return
+	}
+
+	if tok != nil {
+		t.Errorf("Did not expect a token, but got %v", *tok)
+		return
+	}
+}
+
+func TestError(t *testing.T) {
+	l := New("notaspace", WhitespaceState)
+	l.Start()
+
+	tok, done := l.NextToken()
+	if done {
+		t.Error("Expected token to be !done, but it was.")
+		return
+	}
+
+	if tok.Type != ErrorToken {
+		t.Errorf("Expected error token, but got %v", *tok)
+		return
+	}
+}