Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for other log formats #8

Merged
merged 6 commits into from
Jul 29, 2024
Merged

Add support for other log formats #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
59b653f
add failing format test
facundoolano Jul 29, 2024
6b038b4
fix variable name parsing
facundoolano Jul 29, 2024
8770ddc
add iso time field
facundoolano Jul 29, 2024
8331087
add a few more fields
facundoolano Jul 29, 2024
7156443
remove duplicated init
facundoolano Jul 29, 2024
88f2f71
fix mess between competing fields
facundoolano Jul 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion fields.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ var KNOWN_FIELDS = []LogField{
ColumnSpec: "TIMESTAMP NOT NULL",
Parse: parseTime,
},
{
LogFormatVar: "time_iso8601",
ColumnName: "time",
ColumnSpec: "TIMESTAMP NOT NULL",
Parse: parseIsoTime,
},
{
LogFormatVar: "request",
ColumnName: "request_raw",
Expand Down Expand Up @@ -66,13 +72,25 @@ var KNOWN_FIELDS = []LogField{
ColumnName: "status",
ColumnSpec: "INTEGER",
},
{
LogFormatVar: "uri",
CLINames: []string{"path", "url", "uri"},
ColumnName: "path",
ColumnSpec: "TEXT",
},
{
LogFormatVar: "host",
CLINames: []string{"host", "server"},
ColumnName: "host",
ColumnSpec: "TEXT",
},
{
CLINames: []string{"method"},
ColumnName: "method",
ColumnSpec: "TEXT COLLATE NOCASE",
},
{
CLINames: []string{"path", "url"},
CLINames: []string{"path", "url", "uri"},
ColumnName: "path",
ColumnSpec: "TEXT",
},
Expand Down Expand Up @@ -136,6 +154,14 @@ func parseTime(timestamp string) string {
return t.Format(DB_DATE_LAYOUT)
}

func parseIsoTime(timestamp string) string {
t, err := time.Parse("2006-01-02T15:04:05-07:00", timestamp)
if err != nil {
panic("can't parse log timestamp " + timestamp)
}
return t.Format(DB_DATE_LAYOUT)
}

func parseRequestDerivedFields(request string) map[string]string {
result := make(map[string]string)
request_parts := strings.Split(request, " ")
Expand Down
2 changes: 0 additions & 2 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ var NowTimeFun = time.Now
// overridable with NGTOP_LOGS_PATH env var
const DEFAULT_PATH_PATTERN = "/var/log/nginx/access.log*"
const DEFAULT_DB_PATH = "./ngtop.db"

// TODO replace with 'combined' once alias support is added
const DEFAULT_LOG_FORMAT = `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`

func main() {
Expand Down
86 changes: 52 additions & 34 deletions main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,11 @@ xx.xx.xx.xx - - [24/Jul/2024:00:06:41 +0000] "GET /blog/a-note-on-essential-comp
xx.xx.xx.xx - - [24/Jul/2024:00:06:41 +0000] "GET /blog/posdata-de-borges-y-bioy HTTP/1.1" 301 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9"`

func TestBasicQuery(t *testing.T) {
columns, rows := runCommand(t, SAMPLE_LOGS, []string{})
columns, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{})
assertEqual(t, columns, []string{"#reqs"})
assertEqual(t, rows[0][0], "11")

columns, rows = runCommand(t, SAMPLE_LOGS, []string{"url"})
columns, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url"})
assertEqual(t, columns, []string{"path", "#reqs"})
assertEqual(t, len(rows), 5)
assertEqual(t, rows[0], []string{"/feed.xml", "3"})
Expand All @@ -122,36 +122,36 @@ func TestBasicQuery(t *testing.T) {
}

func TestDateFiltering(t *testing.T) {
_, rows := runCommand(t, SAMPLE_LOGS, []string{})
_, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{})
assertEqual(t, rows[0][0], "11")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"-s", "1m"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"-s", "1m"})
assertEqual(t, rows[0][0], "3")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"-u", "1m"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"-u", "1m"})
assertEqual(t, rows[0][0], "8")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"-s", "4m", "-u", "1m"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"-s", "4m", "-u", "1m"})
assertEqual(t, rows[0][0], "1")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"-s", "1h"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"-s", "1h"})
assertEqual(t, rows[0][0], "11")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"-u", "1h"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"-u", "1h"})
assertEqual(t, rows[0][0], "0")
}

func TestLimit(t *testing.T) {
_, rows := runCommand(t, SAMPLE_LOGS, []string{"url"})
_, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url"})
assertEqual(t, len(rows), 5)
_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-l", "3"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-l", "3"})
assertEqual(t, len(rows), 3)
_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-l", "10"})
assertEqual(t, len(rows), 8) // not that many distinct urls
}

func TestMultiField(t *testing.T) {
columns, rows := runCommand(t, SAMPLE_LOGS, []string{"url", "method"})
columns, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "method"})
assertEqual(t, columns, []string{"path", "method", "#reqs"})
assertEqual(t, len(rows), 5)
assertEqual(t, rows[0], []string{"/feed.xml", "GET", "3"})
Expand All @@ -160,85 +160,103 @@ func TestMultiField(t *testing.T) {
assertEqual(t, rows[3][1], "GET")
assertEqual(t, rows[4][1], "GET")

columns, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "status"})
columns, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "status"})
assertEqual(t, columns, []string{"path", "status", "#reqs"})
assertEqual(t, len(rows), 5)
assertEqual(t, rows[0], []string{"/feed.xml", "200", "3"})
assertEqual(t, rows[1], []string{"/feed", "301", "2"})

columns, rows = runCommand(t, SAMPLE_LOGS, []string{"method", "status"})
columns, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"method", "status"})
assertEqual(t, columns, []string{"method", "status", "#reqs"})
assertEqual(t, len(rows), 2)
assertEqual(t, rows[0], []string{"GET", "301", "6"})
assertEqual(t, rows[1], []string{"GET", "200", "5"})

columns, rows = runCommand(t, SAMPLE_LOGS, []string{"status", "method"})
columns, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"status", "method"})
assertEqual(t, columns, []string{"status", "method", "#reqs"})
assertEqual(t, len(rows), 2)
assertEqual(t, rows[0], []string{"301", "GET", "6"})
assertEqual(t, rows[1], []string{"200", "GET", "5"})
}

func TestWhereFilter(t *testing.T) {
columns, rows := runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=200"})
columns, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=200"})
assertEqual(t, columns, []string{"path", "#reqs"})
assertEqual(t, len(rows), 3)
assertEqual(t, rows[0], []string{"/feed.xml", "3"})
assertEqual(t, rows[1][1], "1")
assertEqual(t, rows[2][1], "1")

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=301", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=301", "-l", "10"})
assertEqual(t, len(rows), 5)
_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "method=GET"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "method=GET"})
assertEqual(t, len(rows), 5)
_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "method=get"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "method=get"})
assertEqual(t, len(rows), 5)
}

func TestWhereMultipleValues(t *testing.T) {
_, rows := runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=200", "-w", "status=301"})
_, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=200", "-w", "status=301"})
assertEqual(t, len(rows), 5)
_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=200", "-w", "status=301", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=200", "-w", "status=301", "-l", "10"})
assertEqual(t, len(rows), 8)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi"})
assertEqual(t, len(rows), 2)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi", "-w", "status=200"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi", "-w", "status=200"})
assertEqual(t, len(rows), 1)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi", "-w", "status=200", "-w", "status=301"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "ua=feedi", "-w", "status=200", "-w", "status=301"})
assertEqual(t, len(rows), 2)
}

func TestWherePattern(t *testing.T) {
_, rows := runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "url=/feed%"})
_, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "url=/feed%"})
assertEqual(t, len(rows), 2)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "url=/blog/%"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "url=/blog/%"})
assertEqual(t, len(rows), 5)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=3%"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=3%"})
assertEqual(t, len(rows), 5)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status=2%"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status=2%"})
assertEqual(t, len(rows), 3)
}

func TestWhereNegation(t *testing.T) {
_, rows := runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status!=200", "-l", "10"})
_, rows := runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status!=200", "-l", "10"})
assertEqual(t, len(rows), 5)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status!=301", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status!=301", "-l", "10"})
assertEqual(t, len(rows), 3)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status!=2%", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status!=2%", "-l", "10"})
assertEqual(t, len(rows), 5)

_, rows = runCommand(t, SAMPLE_LOGS, []string{"url", "-w", "status!=3%", "-l", "10"})
_, rows = runCommand(t, DEFAULT_LOG_FORMAT, SAMPLE_LOGS, []string{"url", "-w", "status!=3%", "-l", "10"})
assertEqual(t, len(rows), 3)
}

func TestCustomFormat(t *testing.T) {
format := `$remote_addr [$time_iso8601] $server_name $document_root $host $uri $content_type`
sample := `xx.xx.xx.xx [2024-07-24T00:00:49+00:00] jorge.olano.dev /var/www/jorge jorge.olano.dev /index.html -
xx.xx.xx.xx [2024-07-24T00:00:51+00:00] jorge.olano.dev /var/www/jorge jorge.olano.dev /assets/css/main.css -`
columns, rows := runCommand(t, format, sample, []string{})
assertEqual(t, columns, []string{"#reqs"})
assertEqual(t, rows[0][0], "2")

columns, rows = runCommand(t, format, sample, []string{"uri"})
assertEqual(t, columns, []string{"path", "#reqs"})
assertEqual(t, len(rows), 2)

columns, rows = runCommand(t, format, sample, []string{"host"})
assertEqual(t, columns, []string{"host", "#reqs"})
assertEqual(t, len(rows), 1)
assertEqual(t, rows[0][0], "jorge.olano.dev")
}

func TestMultipleLogFiles(t *testing.T) {
// TODO
// more than one file in a dir, honoring the glob pattern
Expand All @@ -249,7 +267,7 @@ func TestMultipleLogFiles(t *testing.T) {

// ------ HELPERS --------

func runCommand(t *testing.T, logs string, cliArgs []string) ([]string, [][]string) {
func runCommand(t *testing.T, format string, logs string, cliArgs []string) ([]string, [][]string) {
// write the logs to a temp file, and point the NGTOP_LOGS_PATH env to it
logFile, err := os.CreateTemp("", "access.log")
assertEqual(t, err, nil)
Expand All @@ -265,7 +283,7 @@ func runCommand(t *testing.T, logs string, cliArgs []string) ([]string, [][]stri
os.Args = append([]string{"ngtop"}, cliArgs...)
_, spec := querySpecFromCLI()

parser := NewParser(DEFAULT_LOG_FORMAT)
parser := NewParser(format)
dbs, err := InitDB(dbFile.Name(), parser.Fields)
assertEqual(t, err, nil)
defer dbs.Close()
Expand Down
44 changes: 18 additions & 26 deletions parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,6 @@ import (
"time"
)

func init() {
for _, field := range KNOWN_FIELDS {
COLUMN_NAME_TO_FIELD[field.ColumnName] = &field
if field.LogFormatVar != "" {
LOGVAR_TO_FIELD[field.LogFormatVar] = &field
}
for _, name := range field.CLINames {
CLI_NAME_TO_FIELD[name] = &field
}
}
}

const LOG_DATE_LAYOUT = "02/Jan/2006:15:04:05 -0700"

type LogParser struct {
Expand All @@ -39,13 +27,14 @@ func NewParser(format string) *LogParser {
// pick the subset of fields deducted from the regex, plus their derived fields
// use a map to remove duplicates
fieldSubset := make(map[string]*LogField)
for _, name := range parser.formatRegex.SubexpNames() {
if name == "" {
for _, logvar := range parser.formatRegex.SubexpNames() {
if logvar == "" {
continue
}
fieldSubset[name] = COLUMN_NAME_TO_FIELD[name]
field := LOGVAR_TO_FIELD[logvar]
fieldSubset[field.ColumnName] = field

for _, derived := range COLUMN_NAME_TO_FIELD[name].DerivedFields {
for _, derived := range field.DerivedFields {
fieldSubset[derived] = COLUMN_NAME_TO_FIELD[derived]
}
}
Expand Down Expand Up @@ -135,19 +124,18 @@ func formatToRegex(format string) *regexp.Regexp {
} else {
// found a varname, process it
varname := ""
for j := i + 1; j < len(format) && ((chars[j] >= 'a' && chars[j] <= 'z') || chars[j] == '_'); j++ {
for j := i + 1; j < len(format) && isVariableNameRune(chars[j]); j++ {
varname += string(chars[j])
}
i += len(varname)

// write the proper capture group to the format regex pattern
if field, isKnownField := LOGVAR_TO_FIELD[varname]; isKnownField {
if _, isKnownField := LOGVAR_TO_FIELD[varname]; isKnownField {
// if the var matches a field we care to extract, use a named group
groupname := field.ColumnName
if previousWasSpace {
newFormat += "(?P<" + groupname + ">\\S+)"
newFormat += "(?P<" + varname + ">\\S+)"
} else {
newFormat += "(?P<" + groupname + ">.*?)"
newFormat += "(?P<" + varname + ">.*?)"
}
} else {
// otherwise just add a nameless group that ensures matching
Expand All @@ -163,20 +151,24 @@ func formatToRegex(format string) *regexp.Regexp {
return regexp.MustCompile(newFormat)
}

func isVariableNameRune(char rune) bool {
return (char >= 'a' && char <= 'z') || char == '_' || (char >= '0' && char <= '9')
}

func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error) {
match := pattern.FindStringSubmatch(line)
if match == nil {
return nil, fmt.Errorf("log line didn't match format:\nformat:%s\nline:%s", pattern, line)
}

result := make(map[string]string)
for i, name := range pattern.SubexpNames() {
field := COLUMN_NAME_TO_FIELD[name]
if name != "" && match[i] != "-" {
for i, logvar := range pattern.SubexpNames() {
field := LOGVAR_TO_FIELD[logvar]
if logvar != "" && match[i] != "-" {
if field.Parse != nil {
result[name] = field.Parse(match[i])
result[field.ColumnName] = field.Parse(match[i])
} else {
result[name] = match[i]
result[field.ColumnName] = match[i]
}

if field.ParseDerivedFields != nil {
Expand Down