From dbb65f564f89dae06aa854bb5859804cb6d1e99d Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 12:25:40 -0300
Subject: [PATCH 01/20] get log format from environment

---
 accesslog.go | 13 ++++++-------
 main.go      | 14 +++++++++++---
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/accesslog.go b/accesslog.go
index cd6b75e..4c29f75 100644
--- a/accesslog.go
+++ b/accesslog.go
@@ -15,15 +15,14 @@ import (
 	"time"
 )
 
-// TODO add support to nginx config syntax, eg "$remote_addr - $remote_user [$time_local] ..." and add code to translate it to these regexes
-// FIXME consolidate field list (duplicated knowledge)
-const LOG_COMBINED_PATTERN = `(?P<ip>\S+) - (?P<remote_user>\S+) \[(?P<time>.*?)\] "(?P<request_raw>[^"]*)" (?P<status>\d{3}) (?P<bytes_sent>\d+) "(?P<referer>[^"]*)" "(?P<user_agent_raw>[^"]*)"`
-
-var logPattern = regexp.MustCompile(LOG_COMBINED_PATTERN)
-
 // Parse the fields in the nginx access logs since the `until` time, passing them as a map into the `processFun`.
 // Processing is interrupted when a log older than `until` is found.
-func ProcessAccessLogs(logFiles []string, until *time.Time, processFun func(map[string]interface{}) error) error {
+func ProcessAccessLogs(
+	logFormat string,
+	logFiles []string,
+	until *time.Time,
+	processFun func(map[string]interface{}) error,
+) error {
 	for _, path := range logFiles {
 
 		log.Printf("parsing %s", path)
diff --git a/main.go b/main.go
index 3d7c922..076a757 100644
--- a/main.go
+++ b/main.go
@@ -55,6 +55,9 @@ var NowTimeFun = time.Now
 const DEFAULT_PATH_PATTERN = "/var/log/nginx/access.log*"
 const DEFAULT_DB_PATH = "./ngtop.db"
 
+// TODO replace with 'combined' once alias support is added
+const DEFAULT_LOG_FORMAT = `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`
+
 func main() {
 	// Optionally enable internal logger
 	if os.Getenv("NGTOP_LOG") == "" {
@@ -71,12 +74,17 @@ func main() {
 		logPathPattern = envLogsPath
 	}
 
+	logFormat := DEFAULT_LOG_FORMAT
+	if envLogFormat := os.Getenv("NGTOP_LOG_FORMAT"); envLogFormat != "" {
+		logFormat = envLogFormat
+	}
+
 	ctx, spec := querySpecFromCLI()
 	dbs, err := InitDB(dbPath)
 	ctx.FatalIfErrorf(err)
 	defer dbs.Close()
 
-	err = loadLogs(logPathPattern, dbs)
+	err = loadLogs(logFormat, logPathPattern, dbs)
 	ctx.FatalIfErrorf(err)
 
 	columnNames, rowValues, err := dbs.QueryTop(spec)
@@ -187,7 +195,7 @@ func parseDuration(duration string) (time.Time, error) {
 }
 
 // Parse the most recent nginx access.logs and insert the ones not previously seen into the DB.
-func loadLogs(logPathPattern string, dbs *dbSession) error {
+func loadLogs(logFormat string, logPathPattern string, dbs *dbSession) error {
 	logFiles, err := filepath.Glob(logPathPattern)
 	if err != nil {
 		return err
@@ -201,7 +209,7 @@ func loadLogs(logPathPattern string, dbs *dbSession) error {
 		return err
 	}
 
-	err = ProcessAccessLogs(logFiles, lastSeenTime, func(logLineFields map[string]interface{}) error {
+	err = ProcessAccessLogs(logFormat, logFiles, lastSeenTime, func(logLineFields map[string]interface{}) error {
 		queryValues := make([]interface{}, len(dbColumns))
 		for i, field := range dbColumns {
 			queryValues[i] = logLineFields[field]

From de7bc30ff16ee44ba274afae053db571d77e214c Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 13:04:58 -0300
Subject: [PATCH 02/20] stub a lot of changes, but make it compile again

---
 accesslog.go | 107 +++--------------------------------
 db.go        |   2 -
 fields.go    | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++
 main.go      |   2 +-
 4 files changed, 163 insertions(+), 101 deletions(-)
 create mode 100644 fields.go

diff --git a/accesslog.go b/accesslog.go
index 4c29f75..48746da 100644
--- a/accesslog.go
+++ b/accesslog.go
@@ -3,15 +3,10 @@ package main
 import (
 	"bufio"
 	"compress/gzip"
-	"github.com/mileusna/useragent"
 	"io"
 	"log"
-	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
 	"time"
 )
 
@@ -21,8 +16,10 @@ func ProcessAccessLogs(
 	logFormat string,
 	logFiles []string,
 	until *time.Time,
-	processFun func(map[string]interface{}) error,
+	processFun func(map[string]string) error,
 ) error {
+	logPattern := FormatToRegex(logFormat)
+
 	for _, path := range logFiles {
 
 		log.Printf("parsing %s", path)
@@ -43,7 +40,7 @@ func ProcessAccessLogs(
 		scanner := bufio.NewScanner(reader)
 		for scanner.Scan() {
 			line := scanner.Text()
-			values, err := parseLogLine(line)
+			values, err := parseLogLine(logPattern, line)
 			if err != nil {
 				return err
 			}
@@ -52,10 +49,11 @@ func ProcessAccessLogs(
 				continue
 			}
 
-			if until != nil && values["time"].(time.Time).Compare(*until) < 0 {
-				// already caught up, no need to continue processing
-				return nil
-			}
+			// FIXME convert until to string above and make them comparable
+			// if until != nil && values["time"] < *until < 0 {
+			// 	// already caught up, no need to continue processing
+			// 	return nil
+			// }
 
 			if err := processFun(values); err != nil {
 				return err
@@ -69,93 +67,6 @@ func ProcessAccessLogs(
 	return nil
 }
 
-func parseLogLine(logLine string) (map[string]interface{}, error) {
-	match := logPattern.FindStringSubmatch(logLine)
-	if match == nil {
-		return nil, nil
-	}
-	result := make(map[string]interface{})
-	for i, name := range logPattern.SubexpNames() {
-		if i != 0 && name != "" && match[i] != "-" {
-			result[name] = match[i]
-		}
-	}
-
-	// assuming all the fields were found otherwise there would be no match above
-
-	// parse log time to time.Time
-	time, err := timeFromLogFormat(result["time"].(string))
-	if err != nil {
-		return nil, err
-	}
-	result["time"] = time
-
-	// bytes as integer
-	bytes_sent, _ := strconv.Atoi(result["bytes_sent"].(string))
-	result["bytes_sent"] = bytes_sent
-
-	// status as integer
-	status, _ := strconv.Atoi(result["status"].(string))
-	result["status"] = status
-
-	if ua, found := result["user_agent_raw"]; found {
-		ua := useragent.Parse(ua.(string))
-		result["user_agent"] = ua.Name
-		result["os"] = ua.OS
-		result["device"] = ua.Device
-		result["ua_url"] = stripUrlSource(ua.URL)
-		if ua.Bot {
-			result["ua_type"] = "bot"
-		} else if ua.Tablet {
-			result["ua_type"] = "tablet"
-		} else if ua.Mobile {
-			result["ua_type"] = "mobile"
-		} else if ua.Desktop {
-			result["ua_type"] = "desktop"
-		}
-	}
-
-	referer, hasReferer := result["referer"]
-	if hasReferer {
-		result["referer"] = stripUrlSource(referer.(string))
-	}
-
-	request_parts := strings.Split(result["request_raw"].(string), " ")
-	if len(request_parts) == 3 {
-		// if the request line is weird, don't try to extract its fields
-		result["method"] = request_parts[0]
-		raw_path := request_parts[1]
-		if url, err := url.Parse(raw_path); err == nil {
-			result["path"] = url.Path
-
-			// if utm source and friends in query, use them as referer
-			if !hasReferer {
-				keys := []string{"ref", "referer", "referrer", "utm_source"}
-				query := url.Query()
-				for _, key := range keys {
-					if query.Has(key) {
-						result["referer"] = stripUrlSource(query.Get(key))
-						break
-					}
-				}
-			}
-
-		} else {
-			result["path"] = raw_path
-		}
-	}
-
-	return result, nil
-}
-
-func stripUrlSource(value string) string {
-	value = strings.TrimPrefix(value, "http://")
-	value = strings.TrimPrefix(value, "https://")
-	value = strings.TrimPrefix(value, "www.")
-	value = strings.TrimSuffix(value, "/")
-	return value
-}
-
 func timeFromLogFormat(timestamp string) (time.Time, error) {
 	clfLayout := "02/Jan/2006:15:04:05 -0700"
 	return time.Parse(clfLayout, timestamp)
diff --git a/db.go b/db.go
index 96145c5..441dee3 100644
--- a/db.go
+++ b/db.go
@@ -46,7 +46,6 @@ func InitDB(dbPath string) (*dbSession, error) {
 			request_raw		TEXT NOT NULL,
 			user_agent_raw 	TEXT,
 			status			INTEGER,
-			bytes_sent		INTEGER,
 			referer 		TEXT COLLATE NOCASE,
 
 			method			TEXT COLLATE NOCASE,
@@ -57,7 +56,6 @@ func InitDB(dbPath string) (*dbSession, error) {
 			ua_url		 	TEXT,
 			ua_type		 	TEXT COLLATE NOCASE,
 
-
 			created TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 		);
 	`
diff --git a/fields.go b/fields.go
new file mode 100644
index 0000000..f68d431
--- /dev/null
+++ b/fields.go
@@ -0,0 +1,153 @@
+// FIXME this should probably be renamed to parser.go and kill accesslog.log
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"net/url"
+
+	"github.com/mileusna/useragent"
+)
+
+type LogField struct {
+	LogFormatVar       string
+	CLINames           []string
+	ColumnName         string
+	ColumnSpec         string
+	Parse              func(string) string
+	ParseDerivedFields func(string) map[string]string
+}
+
+var KNOWN_FIELDS = []LogField{
+	{
+		LogFormatVar: "$time_local",
+		ColumnName:   "time",
+		ColumnSpec:   "TIMESTAMP NOT NULL",
+		Parse:        parseTime,
+	},
+	{
+		LogFormatVar:       "$request",
+		ColumnName:         "request_raw",
+		ColumnSpec:         "TEXT",
+		ParseDerivedFields: parseRequestDerivedFields,
+	},
+	{
+		LogFormatVar:       "$http_user_agent",
+		ColumnName:         "user_agent_raw",
+		ColumnSpec:         "TEXT",
+		ParseDerivedFields: parseUserAgentDerivedFields,
+	},
+	{
+		LogFormatVar: "$http_referer",
+		CLINames:     []string{"referer", "ref", "referrer"},
+		ColumnName:   "referer",
+		ColumnSpec:   "TEXT COLLATE NOCASE",
+		Parse:        stripUrlSource,
+	},
+
+	// TODO ip
+	// TODO status
+	// TODO method
+	// TODO path
+	// TODO user_agent
+	// TODO os
+	// TODO device
+	// TODO ua_url
+	// TODO ua_type
+}
+
+// FIXME populate in init
+var LOGVAR_TO_NAME = map[string]string{}
+var NAME_TO_FIELD = map[string]LogField{}
+
+func FormatToRegex(format string) *regexp.Regexp {
+
+	// logPattern := regexp.MustCompile(format)
+	return nil
+}
+
+func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error) {
+	match := pattern.FindStringSubmatch(line)
+	if match == nil {
+		return nil, fmt.Errorf("log line didn't match format:\nformat:%s\nline:%s", pattern, line)
+	}
+
+	result := make(map[string]string)
+	for i, name := range pattern.SubexpNames() {
+		if i != 0 && name != "" && match[i] != "-" {
+			result[name] = match[i]
+		}
+	}
+	return result, nil
+}
+
+func stripUrlSource(value string) string {
+	value = strings.TrimPrefix(value, "http://")
+	value = strings.TrimPrefix(value, "https://")
+	value = strings.TrimPrefix(value, "www.")
+	value = strings.TrimSuffix(value, "/")
+	return value
+}
+
+// FIXME error instead of panic?
+func parseTime(timestamp string) string {
+	logLayout := "02/Jan/2006:15:04:05 -0700"
+	dbLayout := "2006-01-02 15:04:05-07:00"
+	t, err := time.Parse(logLayout, timestamp)
+	if err != nil {
+		panic("can't parse log timestamp " + timestamp)
+	}
+	return t.Format(dbLayout)
+}
+
+func parseRequestDerivedFields(request string) map[string]string {
+	result := make(map[string]string)
+	request_parts := strings.Split(request, " ")
+	if len(request_parts) == 3 {
+		// if the request line is weird, don't try to extract its fields
+		result["method"] = request_parts[0]
+		raw_path := request_parts[1]
+		if url, err := url.Parse(raw_path); err == nil {
+			result["path"] = url.Path
+
+			// if utm source and friends in query, use them as referer
+			keys := []string{"ref", "referer", "referrer", "utm_source"}
+			query := url.Query()
+			for _, key := range keys {
+				if query.Has(key) {
+					result["referer"] = stripUrlSource(query.Get(key))
+					break
+				}
+			}
+
+		} else {
+			result["path"] = raw_path
+		}
+	}
+
+	return result
+}
+
+func parseUserAgentDerivedFields(ua string) map[string]string {
+	result := make(map[string]string)
+	if ua != "" {
+		ua := useragent.Parse(ua)
+		result["user_agent"] = ua.Name
+		result["os"] = ua.OS
+		result["device"] = ua.Device
+		result["ua_url"] = stripUrlSource(ua.URL)
+		if ua.Bot {
+			result["ua_type"] = "bot"
+		} else if ua.Tablet {
+			result["ua_type"] = "tablet"
+		} else if ua.Mobile {
+			result["ua_type"] = "mobile"
+		} else if ua.Desktop {
+			result["ua_type"] = "desktop"
+		}
+	}
+	return result
+}
diff --git a/main.go b/main.go
index 076a757..077d024 100644
--- a/main.go
+++ b/main.go
@@ -209,7 +209,7 @@ func loadLogs(logFormat string, logPathPattern string, dbs *dbSession) error {
 		return err
 	}
 
-	err = ProcessAccessLogs(logFormat, logFiles, lastSeenTime, func(logLineFields map[string]interface{}) error {
+	err = ProcessAccessLogs(logFormat, logFiles, lastSeenTime, func(logLineFields map[string]string) error {
 		queryValues := make([]interface{}, len(dbColumns))
 		for i, field := range dbColumns {
 			queryValues[i] = logLineFields[field]

From b905aa4e78df80338a70fe6d285f78eee79587a1 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:00:46 -0300
Subject: [PATCH 03/20] attempt to parse log format strings

---
 fields.go => parser.go | 42 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
 rename fields.go => parser.go (75%)

diff --git a/fields.go b/parser.go
similarity index 75%
rename from fields.go
rename to parser.go
index f68d431..4cbc9ab 100644
--- a/fields.go
+++ b/parser.go
@@ -1,4 +1,3 @@
-// FIXME this should probably be renamed to parser.go and kill accesslog.log
 package main
 
 import (
@@ -64,9 +63,46 @@ var LOGVAR_TO_NAME = map[string]string{}
 var NAME_TO_FIELD = map[string]LogField{}
 
 func FormatToRegex(format string) *regexp.Regexp {
+	return regexp.MustCompile(formatRegexString(format))
+}
 
-	// logPattern := regexp.MustCompile(format)
-	return nil
+// FIXME remove?
+func formatRegexString(format string) string {
+	chars := []rune(format)
+	var newFormat string
+
+	// comesFromSpace is used to tell, when a var is found, if it is expected to contain spaces
+	// without knowing which characters are used to sorround it, eg: "$http_user_agent" [$time_local]
+	// FIXME maybe better to preserve previous char
+	comesFromSpace := true
+	for i := 0; i < len(chars); i++ {
+		if chars[i] != '$' {
+			comesFromSpace = chars[i] == ' '
+			newFormat += regexp.QuoteMeta(string(chars[i]))
+		} else {
+			// found a varname, process it
+			i++
+			varname := ""
+			for ; i < len(format) && ((chars[i] >= 'a' && chars[i] <= 'z') || chars[i] == '_'); i++ {
+				varname += string(chars[i])
+			}
+			if groupname, knownVar := LOGVAR_TO_NAME[varname]; knownVar {
+				if comesFromSpace {
+					newFormat += "(?P<" + groupname + ">\\S+)"
+				} else {
+					newFormat += "(?P<" + groupname + ">.*?)"
+				}
+			} else {
+				if comesFromSpace {
+					newFormat += "(?:\\S+)"
+				} else {
+					newFormat += "(?:.*?)"
+				}
+			}
+
+		}
+	}
+	return newFormat
 }
 
 func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error) {

From 956856ed2317e784724540d20a6428c5f59a664f Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:01:11 -0300
Subject: [PATCH 04/20] prepare test for parser

---
 main_test.go   |  2 +-
 parser_test.go | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 parser_test.go

diff --git a/main_test.go b/main_test.go
index 6f41c75..c670b65 100644
--- a/main_test.go
+++ b/main_test.go
@@ -269,7 +269,7 @@ func runCommand(t *testing.T, logs string, cliArgs []string) ([]string, [][]stri
 	assertEqual(t, err, nil)
 	defer dbs.Close()
 
-	err = loadLogs(logFile.Name(), dbs)
+	err = loadLogs(DEFAULT_LOG_FORMAT, logFile.Name(), dbs)
 	assertEqual(t, err, nil)
 	columnNames, rowValues, err := dbs.QueryTop(spec)
 	assertEqual(t, err, nil)
diff --git a/parser_test.go b/parser_test.go
new file mode 100644
index 0000000..011d2ab
--- /dev/null
+++ b/parser_test.go
@@ -0,0 +1,12 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestFormatRegex(t *testing.T) {
+	formatRegexString(DEFAULT_LOG_FORMAT)
+
+	// FIXME
+	assertEqual(t, 1, 1)
+}

From 9612479b503d68aeeb45a403e1484421627c3b77 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:06:30 -0300
Subject: [PATCH 05/20] fix index moving

---
 parser.go      | 4 ++--
 parser_test.go | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/parser.go b/parser.go
index 4cbc9ab..cb250c6 100644
--- a/parser.go
+++ b/parser.go
@@ -81,11 +81,11 @@ func formatRegexString(format string) string {
 			newFormat += regexp.QuoteMeta(string(chars[i]))
 		} else {
 			// found a varname, process it
-			i++
 			varname := ""
-			for ; i < len(format) && ((chars[i] >= 'a' && chars[i] <= 'z') || chars[i] == '_'); i++ {
+			for j := i + 1; j < len(format) && ((chars[j] >= 'a' && chars[j] <= 'z') || chars[j] == '_'); j++ {
 				varname += string(chars[i])
 			}
+			i += len(varname)
 			if groupname, knownVar := LOGVAR_TO_NAME[varname]; knownVar {
 				if comesFromSpace {
 					newFormat += "(?P<" + groupname + ">\\S+)"
diff --git a/parser_test.go b/parser_test.go
index 011d2ab..b84c815 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -1,12 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"testing"
 )
 
 func TestFormatRegex(t *testing.T) {
-	formatRegexString(DEFAULT_LOG_FORMAT)
+	result := formatRegexString(DEFAULT_LOG_FORMAT)
+	fmt.Printf("LALAL %s", result)
 
 	// FIXME
-	assertEqual(t, 1, 1)
+	assertEqual(t, 1, 2)
 }

From 234f3e9404c7dfc06a092031ef6fddd74730e9cc Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:11:40 -0300
Subject: [PATCH 06/20] fix field log var names

---
 parser.go | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/parser.go b/parser.go
index cb250c6..897452a 100644
--- a/parser.go
+++ b/parser.go
@@ -22,25 +22,25 @@ type LogField struct {
 
 var KNOWN_FIELDS = []LogField{
 	{
-		LogFormatVar: "$time_local",
+		LogFormatVar: "time_local",
 		ColumnName:   "time",
 		ColumnSpec:   "TIMESTAMP NOT NULL",
 		Parse:        parseTime,
 	},
 	{
-		LogFormatVar:       "$request",
+		LogFormatVar:       "request",
 		ColumnName:         "request_raw",
 		ColumnSpec:         "TEXT",
 		ParseDerivedFields: parseRequestDerivedFields,
 	},
 	{
-		LogFormatVar:       "$http_user_agent",
+		LogFormatVar:       "http_user_agent",
 		ColumnName:         "user_agent_raw",
 		ColumnSpec:         "TEXT",
 		ParseDerivedFields: parseUserAgentDerivedFields,
 	},
 	{
-		LogFormatVar: "$http_referer",
+		LogFormatVar: "http_referer",
 		CLINames:     []string{"referer", "ref", "referrer"},
 		ColumnName:   "referer",
 		ColumnSpec:   "TEXT COLLATE NOCASE",
@@ -58,9 +58,17 @@ var KNOWN_FIELDS = []LogField{
 	// TODO ua_type
 }
 
-// FIXME populate in init
 var LOGVAR_TO_NAME = map[string]string{}
-var NAME_TO_FIELD = map[string]LogField{}
+var NAME_TO_FIELD = map[string]*LogField{}
+
+func init() {
+	for _, field := range KNOWN_FIELDS {
+		if field.LogFormatVar != "" {
+			LOGVAR_TO_NAME[field.LogFormatVar] = field.ColumnName
+		}
+		NAME_TO_FIELD[field.ColumnName] = &field
+	}
+}
 
 func FormatToRegex(format string) *regexp.Regexp {
 	return regexp.MustCompile(formatRegexString(format))
@@ -83,9 +91,10 @@ func formatRegexString(format string) string {
 			// found a varname, process it
 			varname := ""
 			for j := i + 1; j < len(format) && ((chars[j] >= 'a' && chars[j] <= 'z') || chars[j] == '_'); j++ {
-				varname += string(chars[i])
+				varname += string(chars[j])
 			}
 			i += len(varname)
+			fmt.Printf("LALA variable is %s\n", varname)
 			if groupname, knownVar := LOGVAR_TO_NAME[varname]; knownVar {
 				if comesFromSpace {
 					newFormat += "(?P<" + groupname + ">\\S+)"

From 78336dbafe09dfa730ff30d7f6bad65e286d373f Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:48:20 -0300
Subject: [PATCH 07/20] passing test with basic fields

---
 parser.go      |  2 +-
 parser_test.go | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/parser.go b/parser.go
index 897452a..dbdbef3 100644
--- a/parser.go
+++ b/parser.go
@@ -74,7 +74,7 @@ func FormatToRegex(format string) *regexp.Regexp {
 	return regexp.MustCompile(formatRegexString(format))
 }
 
-// FIXME remove?
+// FIXME this separation is only temporary, move to the one above
 func formatRegexString(format string) string {
 	chars := []rune(format)
 	var newFormat string
diff --git a/parser_test.go b/parser_test.go
index b84c815..337b7ba 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -1,14 +1,18 @@
 package main
 
 import (
-	"fmt"
 	"testing"
 )
 
 func TestFormatRegex(t *testing.T) {
-	result := formatRegexString(DEFAULT_LOG_FORMAT)
-	fmt.Printf("LALAL %s", result)
+	line := `xx.xx.xx.xx - - [24/Jul/2024:00:00:28 +0000] "GET /feed HTTP/1.1" 301 169 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"`
 
-	// FIXME
-	assertEqual(t, 1, 2)
+	pattern := FormatToRegex(DEFAULT_LOG_FORMAT)
+	result, err := parseLogLine(pattern, line)
+	assertEqual(t, err, nil)
+
+	assertEqual(t, result["time"], "24/Jul/2024:00:00:28 +0000")
+	assertEqual(t, result["request_raw"], "GET /feed HTTP/1.1")
+	assertEqual(t, result["user_agent_raw"], "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
+	assertEqual(t, result["referer"], "")
 }

From 647b7eacfa5682b3b1dca3d8a318bd7ef8ee168b Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:49:12 -0300
Subject: [PATCH 08/20] remove unnecessary helper function

---
 parser.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/parser.go b/parser.go
index dbdbef3..9326987 100644
--- a/parser.go
+++ b/parser.go
@@ -71,11 +71,6 @@ func init() {
 }
 
 func FormatToRegex(format string) *regexp.Regexp {
-	return regexp.MustCompile(formatRegexString(format))
-}
-
-// FIXME this separation is only temporary, move to the one above
-func formatRegexString(format string) string {
 	chars := []rune(format)
 	var newFormat string
 
@@ -111,7 +106,7 @@ func formatRegexString(format string) string {
 
 		}
 	}
-	return newFormat
+	return regexp.MustCompile(newFormat)
 }
 
 func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error) {

From 022c124a76d3f12ef81fb22dcbb887f93a41a002 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 14:54:40 -0300
Subject: [PATCH 09/20] cleanup

---
 parser.go | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/parser.go b/parser.go
index 9326987..62e9575 100644
--- a/parser.go
+++ b/parser.go
@@ -70,17 +70,17 @@ func init() {
 	}
 }
 
+// TODO
 func FormatToRegex(format string) *regexp.Regexp {
 	chars := []rune(format)
 	var newFormat string
 
-	// comesFromSpace is used to tell, when a var is found, if it is expected to contain spaces
+	// previousWasSpace is used to tell, when a var is found, if it is expected to contain spaces
 	// without knowing which characters are used to sorround it, eg: "$http_user_agent" [$time_local]
-	// FIXME maybe better to preserve previous char
-	comesFromSpace := true
+	previousWasSpace := true
 	for i := 0; i < len(chars); i++ {
 		if chars[i] != '$' {
-			comesFromSpace = chars[i] == ' '
+			previousWasSpace = chars[i] == ' '
 			newFormat += regexp.QuoteMeta(string(chars[i]))
 		} else {
 			// found a varname, process it
@@ -89,15 +89,18 @@ func FormatToRegex(format string) *regexp.Regexp {
 				varname += string(chars[j])
 			}
 			i += len(varname)
-			fmt.Printf("LALA variable is %s\n", varname)
-			if groupname, knownVar := LOGVAR_TO_NAME[varname]; knownVar {
-				if comesFromSpace {
+
+			// write the proper capture group to the format regex pattern
+			if groupname, isKnownField := LOGVAR_TO_NAME[varname]; isKnownField {
+				// if the var matches a field we care to extract, use a named group
+				if previousWasSpace {
 					newFormat += "(?P<" + groupname + ">\\S+)"
 				} else {
 					newFormat += "(?P<" + groupname + ">.*?)"
 				}
 			} else {
-				if comesFromSpace {
+				// otherwise just add a nameless group that ensures matching
+				if previousWasSpace {
 					newFormat += "(?:\\S+)"
 				} else {
 					newFormat += "(?:.*?)"

From 8ed48e12112f0645ea8d2dd8160a832d04d0d751 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 15:07:10 -0300
Subject: [PATCH 10/20] add parsing of derived fields

---
 parser.go      | 13 ++++++++++++-
 parser_test.go |  6 +++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/parser.go b/parser.go
index 62e9575..bb2eb23 100644
--- a/parser.go
+++ b/parser.go
@@ -120,8 +120,19 @@ func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error
 
 	result := make(map[string]string)
 	for i, name := range pattern.SubexpNames() {
+		field := NAME_TO_FIELD[name]
 		if i != 0 && name != "" && match[i] != "-" {
-			result[name] = match[i]
+			if field.Parse != nil {
+				result[name] = field.Parse(match[i])
+			} else {
+				result[name] = match[i]
+			}
+
+			if field.ParseDerivedFields != nil {
+				for key, value := range field.ParseDerivedFields(match[i]) {
+					result[key] = value
+				}
+			}
 		}
 	}
 	return result, nil
diff --git a/parser_test.go b/parser_test.go
index 337b7ba..e715df2 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -11,8 +11,12 @@ func TestFormatRegex(t *testing.T) {
 	result, err := parseLogLine(pattern, line)
 	assertEqual(t, err, nil)
 
-	assertEqual(t, result["time"], "24/Jul/2024:00:00:28 +0000")
+	assertEqual(t, result["time"], "2024-07-24 00:00:28+00:00")
 	assertEqual(t, result["request_raw"], "GET /feed HTTP/1.1")
 	assertEqual(t, result["user_agent_raw"], "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
 	assertEqual(t, result["referer"], "")
+
+	// derived fields
+	assertEqual(t, result["path"], "/feed")
+	assertEqual(t, result["method"], "GET")
 }

From 7bf62afae1237716dc59cd1c853e2efd2d0c0ba2 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 15:13:43 -0300
Subject: [PATCH 11/20] make index maps more consistent

---
 accesslog.go |  1 +
 db.go        |  2 +-
 parser.go    | 17 +++++++++++------
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/accesslog.go b/accesslog.go
index 48746da..5bc79d4 100644
--- a/accesslog.go
+++ b/accesslog.go
@@ -1,3 +1,4 @@
+// FIXME this file shouldn't exist
 package main
 
 import (
diff --git a/db.go b/db.go
index 441dee3..5e8f8ce 100644
--- a/db.go
+++ b/db.go
@@ -36,7 +36,7 @@ func InitDB(dbPath string) (*dbSession, error) {
 	}
 
 	// TODO consider adding indexes according to expected queries
-
+	// FIXME build this dynamically based on the log format columns
 	sqlStmt := `
 		CREATE TABLE IF NOT EXISTS access_logs (
 			id 				INTEGER NOT NULL PRIMARY KEY,
diff --git a/parser.go b/parser.go
index bb2eb23..41cd7aa 100644
--- a/parser.go
+++ b/parser.go
@@ -58,15 +58,19 @@ var KNOWN_FIELDS = []LogField{
 	// TODO ua_type
 }
 
-var LOGVAR_TO_NAME = map[string]string{}
-var NAME_TO_FIELD = map[string]*LogField{}
+var LOGVAR_TO_FIELD = map[string]*LogField{}
+var COLUMN_NAME_TO_FIELD = map[string]*LogField{}
+var CLI_NAME_TO_FIELD = map[string]*LogField{}
 
 func init() {
 	for _, field := range KNOWN_FIELDS {
+		COLUMN_NAME_TO_FIELD[field.ColumnName] = &field
 		if field.LogFormatVar != "" {
-			LOGVAR_TO_NAME[field.LogFormatVar] = field.ColumnName
+			LOGVAR_TO_FIELD[field.LogFormatVar] = &field
+		}
+		for _, name := range field.CLINames {
+			CLI_NAME_TO_FIELD[name] = &field
 		}
-		NAME_TO_FIELD[field.ColumnName] = &field
 	}
 }
 
@@ -91,8 +95,9 @@ func FormatToRegex(format string) *regexp.Regexp {
 			i += len(varname)
 
 			// write the proper capture group to the format regex pattern
-			if groupname, isKnownField := LOGVAR_TO_NAME[varname]; isKnownField {
+			if field, isKnownField := LOGVAR_TO_FIELD[varname]; isKnownField {
 				// if the var matches a field we care to extract, use a named group
+				groupname := field.ColumnName
 				if previousWasSpace {
 					newFormat += "(?P<" + groupname + ">\\S+)"
 				} else {
@@ -120,7 +125,7 @@ func parseLogLine(pattern *regexp.Regexp, line string) (map[string]string, error
 
 	result := make(map[string]string)
 	for i, name := range pattern.SubexpNames() {
-		field := NAME_TO_FIELD[name]
+		field := COLUMN_NAME_TO_FIELD[name]
 		if i != 0 && name != "" && match[i] != "-" {
 			if field.Parse != nil {
 				result[name] = field.Parse(match[i])

From 68ae5ca3f9d8735e10daf10cea35d64f2e2b6836 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:26:08 -0300
Subject: [PATCH 12/20] add remaining fields

---
 db.go          |  4 ++--
 parser.go      | 57 +++++++++++++++++++++++++++++++++++++++++---------
 parser_test.go |  9 ++++++--
 3 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/db.go b/db.go
index 5e8f8ce..bbdcf75 100644
--- a/db.go
+++ b/db.go
@@ -40,6 +40,7 @@ func InitDB(dbPath string) (*dbSession, error) {
 	sqlStmt := `
 		CREATE TABLE IF NOT EXISTS access_logs (
 			id 				INTEGER NOT NULL PRIMARY KEY,
+			created TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 
 			ip				TEXT,
 			time 			TIMESTAMP NOT NULL,
@@ -54,9 +55,8 @@ func InitDB(dbPath string) (*dbSession, error) {
 			os			 	TEXT COLLATE NOCASE,
 			device		 	TEXT COLLATE NOCASE,
 			ua_url		 	TEXT,
-			ua_type		 	TEXT COLLATE NOCASE,
+			ua_type		 	TEXT COLLATE NOCASE
 
-			created TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 		);
 	`
 	_, err = db.Exec(sqlStmt)
diff --git a/parser.go b/parser.go
index 41cd7aa..1850173 100644
--- a/parser.go
+++ b/parser.go
@@ -46,16 +46,53 @@ var KNOWN_FIELDS = []LogField{
 		ColumnSpec:   "TEXT COLLATE NOCASE",
 		Parse:        stripUrlSource,
 	},
-
-	// TODO ip
-	// TODO status
-	// TODO method
-	// TODO path
-	// TODO user_agent
-	// TODO os
-	// TODO device
-	// TODO ua_url
-	// TODO ua_type
+	{
+		LogFormatVar: "remote_addr",
+		CLINames:     []string{"ip"},
+		ColumnName:   "ip",
+		ColumnSpec:   "TEXT",
+	},
+	{
+		LogFormatVar: "status",
+		CLINames:     []string{"status"},
+		ColumnName:   "status",
+		ColumnSpec:   "INTEGER",
+	},
+	{
+		CLINames:   []string{"method"},
+		ColumnName: "method",
+		ColumnSpec: "TEXT COLLATE NOCASE",
+	},
+	{
+		CLINames:   []string{"path", "url"},
+		ColumnName: "path",
+		ColumnSpec: "TEXT",
+	},
+	{
+		CLINames:   []string{"user_agent", "ua", "useragent"},
+		ColumnName: "user_agent",
+		ColumnSpec: "TEXT COLLATE NOCASE",
+	},
+	{
+		CLINames:   []string{"os"},
+		ColumnName: "os",
+		ColumnSpec: "TEXT COLLATE NOCASE",
+	},
+	{
+		CLINames:   []string{"device"},
+		ColumnName: "device",
+		ColumnSpec: "TEXT COLLATE NOCASE",
+	},
+	{
+		CLINames:   []string{"ua_url", "uaurl"},
+		ColumnName: "ua_url",
+		ColumnSpec: "TEXT",
+	},
+	{
+		CLINames:   []string{"ua_type", "uatype"},
+		ColumnName: "ua_type",
+		ColumnSpec: "TEXT COLLATE NOCASE",
+	},
 }
 
 var LOGVAR_TO_FIELD = map[string]*LogField{}
diff --git a/parser_test.go b/parser_test.go
index e715df2..2f109fd 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -11,12 +11,17 @@ func TestFormatRegex(t *testing.T) {
 	result, err := parseLogLine(pattern, line)
 	assertEqual(t, err, nil)
 
+	assertEqual(t, result["ip"], "xx.xx.xx.xx")
 	assertEqual(t, result["time"], "2024-07-24 00:00:28+00:00")
 	assertEqual(t, result["request_raw"], "GET /feed HTTP/1.1")
-	assertEqual(t, result["user_agent_raw"], "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
+	assertEqual(t, result["status"], "301")
 	assertEqual(t, result["referer"], "")
+	assertEqual(t, result["user_agent_raw"], "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
 
 	// derived fields
-	assertEqual(t, result["path"], "/feed")
 	assertEqual(t, result["method"], "GET")
+	assertEqual(t, result["path"], "/feed")
+	assertEqual(t, result["user_agent"], "Chrome")
+	assertEqual(t, result["ua_type"], "desktop")
+	assertEqual(t, result["os"], "Windows")
 }

From 7fa18de98819473e62fad0f60295b2d4650e668f Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:27:54 -0300
Subject: [PATCH 13/20] remove unused bytes field

---
 main.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/main.go b/main.go
index 077d024..7d25a3d 100644
--- a/main.go
+++ b/main.go
@@ -36,8 +36,6 @@ var FIELD_NAMES = map[string]string{
 	"os":         "os",
 	"device":     "device",
 	"request":    "request_raw",
-	"bytes":      "bytes_sent",
-	"bytes_sent": "bytes_sent",
 	"path":       "path",
 	"url":        "path",
 	"ip":         "ip",
@@ -202,7 +200,7 @@ func loadLogs(logFormat string, logPathPattern string, dbs *dbSession) error {
 	}
 
 	// FIXME consolidate field list (duplicated knowledge)
-	dbColumns := []string{"ip", "time", "request_raw", "status", "bytes_sent", "referer", "user_agent_raw", "method", "path", "user_agent", "os", "device", "ua_url", "ua_type"}
+	dbColumns := []string{"ip", "time", "request_raw", "status", "referer", "user_agent_raw", "method", "path", "user_agent", "os", "device", "ua_url", "ua_type"}
 
 	lastSeenTime, err := dbs.PrepareForUpdate(dbColumns)
 	if err != nil {

From 67bb66973a9f5039ce37852870a40846faee135b Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:37:08 -0300
Subject: [PATCH 14/20] restore until date check

---
 accesslog.go | 18 +++++++++---------
 db.go        |  9 +++------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/accesslog.go b/accesslog.go
index 5bc79d4..811776e 100644
--- a/accesslog.go
+++ b/accesslog.go
@@ -21,6 +21,11 @@ func ProcessAccessLogs(
 ) error {
 	logPattern := FormatToRegex(logFormat)
 
+	var untilStr string
+	if until != nil {
+		until.Format(DB_DATE_LAYOUT)
+	}
+
 	for _, path := range logFiles {
 
 		log.Printf("parsing %s", path)
@@ -51,10 +56,10 @@ func ProcessAccessLogs(
 			}
 
 			// FIXME convert until to string above and make them comparable
-			// if until != nil && values["time"] < *until < 0 {
-			// 	// already caught up, no need to continue processing
-			// 	return nil
-			// }
+			if untilStr != "" && values["time"] < untilStr {
+				// already caught up, no need to continue processing
+				return nil
+			}
 
 			if err := processFun(values); err != nil {
 				return err
@@ -67,8 +72,3 @@ func ProcessAccessLogs(
 
 	return nil
 }
-
-func timeFromLogFormat(timestamp string) (time.Time, error) {
-	clfLayout := "02/Jan/2006:15:04:05 -0700"
-	return time.Parse(clfLayout, timestamp)
-}
diff --git a/db.go b/db.go
index bbdcf75..c5c74f1 100644
--- a/db.go
+++ b/db.go
@@ -24,6 +24,8 @@ type dbSession struct {
 	insertStmt *sql.Stmt
 }
 
+const DB_DATE_LAYOUT = "2006-01-02 15:04:05-07:00"
+
 // Open or create the database at the given path.
 func InitDB(dbPath string) (*dbSession, error) {
 	db, err := sql.Open("sqlite3", dbPath)
@@ -84,7 +86,7 @@ func (dbs *dbSession) PrepareForUpdate(columns []string) (*time.Time, error) {
 			return nil, err
 		}
 
-		t, _ := timeFromDBFormat(lastSeenTimeStr)
+		t, _ := time.Parse(DB_DATE_LAYOUT, lastSeenTimeStr)
 		lastSeemTime = &t
 	}
 
@@ -215,8 +217,3 @@ func (spec *RequestCountSpec) buildQuery() (string, []any) {
 
 	return queryString, queryArgs
 }
-
-func timeFromDBFormat(timestamp string) (time.Time, error) {
-	sqliteLayout := "2006-01-02 15:04:05-07:00"
-	return time.Parse(sqliteLayout, timestamp)
-}

From 9c5a0b115600f702fe30ccf1ec0be4a245a9e316 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:37:41 -0300
Subject: [PATCH 15/20] remove outdated comment

---
 accesslog.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/accesslog.go b/accesslog.go
index 811776e..a531c48 100644
--- a/accesslog.go
+++ b/accesslog.go
@@ -55,7 +55,6 @@ func ProcessAccessLogs(
 				continue
 			}
 
-			// FIXME convert until to string above and make them comparable
 			if untilStr != "" && values["time"] < untilStr {
 				// already caught up, no need to continue processing
 				return nil

From 9f149ae27364e62924a6cf308c12c9f726cb2c5b Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:40:45 -0300
Subject: [PATCH 16/20] remove accesslogs.go

---
 accesslog.go   | 73 --------------------------------------------------
 parser.go      | 69 ++++++++++++++++++++++++++++++++++++++++++++++-
 parser_test.go |  2 +-
 3 files changed, 69 insertions(+), 75 deletions(-)
 delete mode 100644 accesslog.go

diff --git a/accesslog.go b/accesslog.go
deleted file mode 100644
index a531c48..0000000
--- a/accesslog.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// FIXME this file shouldn't exist
-package main
-
-import (
-	"bufio"
-	"compress/gzip"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-// Parse the fields in the nginx access logs since the `until` time, passing them as a map into the `processFun`.
-// Processing is interrupted when a log older than `until` is found.
-func ProcessAccessLogs(
-	logFormat string,
-	logFiles []string,
-	until *time.Time,
-	processFun func(map[string]string) error,
-) error {
-	logPattern := FormatToRegex(logFormat)
-
-	var untilStr string
-	if until != nil {
-		until.Format(DB_DATE_LAYOUT)
-	}
-
-	for _, path := range logFiles {
-
-		log.Printf("parsing %s", path)
-		file, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-
-		// if it's gzipped, wrap in a decompressing reader
-		var reader io.Reader = file
-		if filepath.Ext(path) == ".gz" {
-			if reader, err = gzip.NewReader(file); err != nil {
-				return err
-			}
-		}
-
-		scanner := bufio.NewScanner(reader)
-		for scanner.Scan() {
-			line := scanner.Text()
-			values, err := parseLogLine(logPattern, line)
-			if err != nil {
-				return err
-			}
-			if values == nil {
-				log.Printf("couldn't parse line %s", line)
-				continue
-			}
-
-			if untilStr != "" && values["time"] < untilStr {
-				// already caught up, no need to continue processing
-				return nil
-			}
-
-			if err := processFun(values); err != nil {
-				return err
-			}
-		}
-		if err := scanner.Err(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
diff --git a/parser.go b/parser.go
index 1850173..a30f1bb 100644
--- a/parser.go
+++ b/parser.go
@@ -1,7 +1,13 @@
 package main
 
 import (
+	"bufio"
+	"compress/gzip"
 	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
@@ -111,8 +117,69 @@ func init() {
 	}
 }
 
+// Parse the fields in the nginx access logs since the `until` time, passing them as a map into the `processFun`.
+// Processing is interrupted when a log older than `until` is found.
+func ProcessAccessLogs(
+	logFormat string,
+	logFiles []string,
+	until *time.Time,
+	processFun func(map[string]string) error,
+) error {
+	logPattern := formatToRegex(logFormat)
+
+	var untilStr string
+	if until != nil {
+		until.Format(DB_DATE_LAYOUT)
+	}
+
+	for _, path := range logFiles {
+
+		log.Printf("parsing %s", path)
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		// if it's gzipped, wrap in a decompressing reader
+		var reader io.Reader = file
+		if filepath.Ext(path) == ".gz" {
+			if reader, err = gzip.NewReader(file); err != nil {
+				return err
+			}
+		}
+
+		scanner := bufio.NewScanner(reader)
+		for scanner.Scan() {
+			line := scanner.Text()
+			values, err := parseLogLine(logPattern, line)
+			if err != nil {
+				return err
+			}
+			if values == nil {
+				log.Printf("couldn't parse line %s", line)
+				continue
+			}
+
+			if untilStr != "" && values["time"] < untilStr {
+				// already caught up, no need to continue processing
+				return nil
+			}
+
+			if err := processFun(values); err != nil {
+				return err
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // TODO
-func FormatToRegex(format string) *regexp.Regexp {
+func formatToRegex(format string) *regexp.Regexp {
 	chars := []rune(format)
 	var newFormat string
 
diff --git a/parser_test.go b/parser_test.go
index 2f109fd..1f2eb3e 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -7,7 +7,7 @@ import (
 func TestFormatRegex(t *testing.T) {
 	line := `xx.xx.xx.xx - - [24/Jul/2024:00:00:28 +0000] "GET /feed HTTP/1.1" 301 169 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"`
 
-	pattern := FormatToRegex(DEFAULT_LOG_FORMAT)
+	pattern := formatToRegex(DEFAULT_LOG_FORMAT)
 	result, err := parseLogLine(pattern, line)
 	assertEqual(t, err, nil)
 

From 7080947ad03e2677428bf313a4bec309ea37b11d Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:41:27 -0300
Subject: [PATCH 17/20] fix bug

---
 parser.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser.go b/parser.go
index a30f1bb..6c1b6e2 100644
--- a/parser.go
+++ b/parser.go
@@ -129,7 +129,7 @@ func ProcessAccessLogs(
 
 	var untilStr string
 	if until != nil {
-		until.Format(DB_DATE_LAYOUT)
+		untilStr = until.Format(DB_DATE_LAYOUT)
 	}
 
 	for _, path := range logFiles {

From 535d7617f6744a6303d0863816e08c2ca1bbbf5b Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:47:10 -0300
Subject: [PATCH 18/20] todo

---
 parser.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/parser.go b/parser.go
index 6c1b6e2..93c5d0d 100644
--- a/parser.go
+++ b/parser.go
@@ -17,15 +17,23 @@ import (
 	"github.com/mileusna/useragent"
 )
 
+// TODO
 type LogField struct {
-	LogFormatVar       string
-	CLINames           []string
-	ColumnName         string
-	ColumnSpec         string
-	Parse              func(string) string
+	// TODO
+	LogFormatVar string
+	// TODO
+	CLINames []string
+	// TODO
+	ColumnName string
+	// TODO
+	ColumnSpec string
+	// TODO
+	Parse func(string) string
+	// TODO
 	ParseDerivedFields func(string) map[string]string
 }
 
+// TODO
 var KNOWN_FIELDS = []LogField{
 	{
 		LogFormatVar: "time_local",

From 659c8dc990973e4e471408d5f3804dcc18b738a9 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:52:29 -0300
Subject: [PATCH 19/20] more date constants

---
 parser.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser.go b/parser.go
index 93c5d0d..7977f58 100644
--- a/parser.go
+++ b/parser.go
@@ -125,6 +125,8 @@ func init() {
 	}
 }
 
+const LOG_DATE_LAYOUT = "02/Jan/2006:15:04:05 -0700"
+
 // Parse the fields in the nginx access logs since the `until` time, passing them as a map into the `processFun`.
 // Processing is interrupted when a log older than `until` is found.
 func ProcessAccessLogs(
@@ -265,13 +267,11 @@ func stripUrlSource(value string) string {
 
 // FIXME error instead of panic?
 func parseTime(timestamp string) string {
-	logLayout := "02/Jan/2006:15:04:05 -0700"
-	dbLayout := "2006-01-02 15:04:05-07:00"
-	t, err := time.Parse(logLayout, timestamp)
+	t, err := time.Parse(LOG_DATE_LAYOUT, timestamp)
 	if err != nil {
 		panic("can't parse log timestamp " + timestamp)
 	}
-	return t.Format(dbLayout)
+	return t.Format(DB_DATE_LAYOUT)
 }
 
 func parseRequestDerivedFields(request string) map[string]string {

From a7f2212e3a8eaf028d879f96156f5aa7a3002a81 Mon Sep 17 00:00:00 2001
From: facundo <facundo.olano@gmail.com>
Date: Sun, 28 Jul 2024 16:58:40 -0300
Subject: [PATCH 20/20] remove redundante FIELD_NAMES map

---
 main.go | 33 ++++++---------------------------
 1 file changed, 6 insertions(+), 27 deletions(-)

diff --git a/main.go b/main.go
index 7d25a3d..6974772 100644
--- a/main.go
+++ b/main.go
@@ -24,27 +24,6 @@ type CommandArgs struct {
 	Where  []string `short:"w" optional:"" help:"Filter expressions. Example: -w useragent=Safari -w status=200"`
 }
 
-// FIXME consolidate field list (duplicated knowledge)
-var FIELD_NAMES = map[string]string{
-	"user_agent": "user_agent",
-	"useragent":  "user_agent",
-	"ua":         "user_agent",
-	"ua_type":    "ua_type",
-	"uatype":     "ua_type",
-	"ua_url":     "ua_url",
-	"uaurl":      "ua_url",
-	"os":         "os",
-	"device":     "device",
-	"request":    "request_raw",
-	"path":       "path",
-	"url":        "path",
-	"ip":         "ip",
-	"referer":    "referer",
-	"referrer":   "referer",
-	"status":     "status",
-	"method":     "method",
-}
-
 // Use a var to get current time, allowing for tests to override it
 var NowTimeFun = time.Now
 
@@ -93,8 +72,8 @@ func main() {
 // Parse the command line arguments into a top requests query specification
 func querySpecFromCLI() (*kong.Context, *RequestCountSpec) {
 	// Parse query spec first, i.e. don't bother with db updates if the command is invalid
-	fieldNames := make([]string, 0, len(FIELD_NAMES))
-	for k := range FIELD_NAMES {
+	fieldNames := make([]string, 0, len(CLI_NAME_TO_FIELD))
+	for k := range CLI_NAME_TO_FIELD {
 		fieldNames = append(fieldNames, k)
 	}
 
@@ -117,7 +96,7 @@ func querySpecFromCLI() (*kong.Context, *RequestCountSpec) {
 	// translate field name aliases
 	columns := make([]string, len(cli.Fields))
 	for i, field := range cli.Fields {
-		columns[i] = FIELD_NAMES[field]
+		columns[i] = CLI_NAME_TO_FIELD[field].ColumnName
 	}
 
 	whereConditions, err := resolveWhereConditions(cli.Where)
@@ -150,10 +129,10 @@ func resolveWhereConditions(clauses []string) (map[string][]string, error) {
 			return nil, fmt.Errorf("invalid where expression %s", clause)
 		}
 
-		if column, found := FIELD_NAMES[keyvalue[0]]; !found {
-			return nil, fmt.Errorf("unknown field name %s", keyvalue[0])
+		if field, found := CLI_NAME_TO_FIELD[keyvalue[0]]; found {
+			conditions[field.ColumnName] = append(conditions[field.ColumnName], keyvalue[1])
 		} else {
-			conditions[column] = append(conditions[column], keyvalue[1])
+			return nil, fmt.Errorf("unknown field name %s", keyvalue[0])
 		}
 	}