-
Notifications
You must be signed in to change notification settings - Fork 0
/
lexer.go
96 lines (83 loc) · 1.91 KB
/
lexer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package golas
import (
"bufio"
"bytes"
"io"
)
// HandlerFunc func used as lexer state
type HandlerFunc func(*Lexer) HandlerFunc
// Lexer can perform lexical analysis on .las files
type Lexer struct {
buffer *bytes.Buffer
char rune
dots int
handler HandlerFunc
line int
position int
reader *bufio.Reader
tokens chan Token
}
// NewLexer creates a new Lexer
func NewLexer(r io.Reader) Lexer {
return Lexer{
reader: bufio.NewReader(r),
tokens: make(chan Token, 3),
buffer: &bytes.Buffer{},
}
}
// NextToken reads the next token from our tokens chan
func (l *Lexer) NextToken() Token {
for {
select {
case token := <-l.tokens:
return token
default:
if l.handler == nil {
panic("private field `Lexer.handler` is nil : unable to acquire token without state")
}
l.handler = l.handler(l)
}
}
}
// Start sets our handler, in turn starting lexical analysis
func (l *Lexer) Start(hf HandlerFunc) {
l.handler = hf
}
// emit places a token on our tokens chan
func (l *Lexer) emit(t TokenType) {
l.tokens <- Token{t, l.buffer.String()}
l.buffer.Reset()
}
// overwriteBuffer clears our buffer then writes a string to it
func (l *Lexer) overwriteBuffer(s string) {
l.buffer.Reset()
l.buffer.WriteString(s)
}
// step consumes the next rune from our reader
func (l *Lexer) step() {
ch, _, err := l.reader.ReadRune()
if err != nil {
ch = -1 // Use -1 to signal EOF
}
// If no error, increment position before moving on
l.position++
switch ch {
case '\n':
l.line++
l.position, l.dots = 0, 0
case '.':
l.dots = l.dots + 1
}
l.buffer.WriteRune(ch)
l.char = ch
}
// stepUntil reads from current line position until we read a certain rune
func (l *Lexer) stepUntil(char rune) {
for l.char != char {
l.step()
}
}
// truncate our buffer by 1. If our buffer were a string, this removes the last character
func (l *Lexer) truncate() {
l.buffer.Truncate(l.buffer.Len() - 1)
}