forked from octacian/timecalc
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokenize.go
94 lines (78 loc) · 2.2 KB
/
tokenize.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package main
import (
"fmt"
)
// Token represents a part of some user input
type Token struct {
Type string
Value string
}
// Tokenize a string, returning an array of tokens
func Tokenize(str string) ([]*Token, error) {
tokens := make([]*Token, 0)
last := ""
openGroupCount := 0
closeGroupCount := 0
for i, v := range str {
c := string(v)
var next string
if i+1 < len(str) {
next = string(str[i+1])
}
if OperatorRegex.MatchString(c) {
tokens = append(tokens, &Token{"operator", c})
if OperatorRegex.MatchString(next) {
return nil, fmt.Errorf("tokenize: unexpected operator")
}
} else if NumberRegex.MatchString(c) {
if NumberRegex.MatchString(next) || next == ":" || next == "." {
last += c
continue
} else if last != "" {
last += c
}
if last == "" {
tokens = append(tokens, &Token{"number", c})
}
} else if c == ":" || c == "." {
last += c
} else if c == "(" || c == "[" {
tokens = append(tokens, &Token{"groupOpen", c})
openGroupCount++
} else if c == ")" || c == "]" {
tokens = append(tokens, &Token{"groupClose", c})
closeGroupCount++
} else if WhitespaceRegex.MatchString(c) {
tokens = append(tokens, &Token{"whitespace", c})
} else {
return tokens, fmt.Errorf("tokenize: invalid charactor: %v", c)
}
if last != "" && !(NumberRegex.MatchString(next) || next == ":" || next == ".") {
if NumberRegex.MatchString(last) {
tokens = append(tokens, &Token{"number", last})
} else if TimeRegex.MatchString(last) {
tokens = append(tokens, &Token{"time", last})
} else {
return nil, fmt.Errorf("tokenize: invalid time or number")
}
last = ""
}
}
// if an unequal number of group openings and closings were found, return error
if openGroupCount != closeGroupCount {
return nil, fmt.Errorf("tokenize: uneqal number of group openings and closings found (%d opened, "+
"%d closed)", openGroupCount, closeGroupCount)
}
return tokens, nil
}
// Reconstruct a string from a token list
func Reconstruct(tokens []*Token, verbose bool) string {
out := ""
for _, token := range tokens {
out += token.Value
if verbose && token.Type != "whitespace" {
out += " (" + token.Type + ")"
}
}
return out
}