From edc66454d049141b70e3140a72e52ead5ce18250 Mon Sep 17 00:00:00 2001 From: Facundo Olano Date: Tue, 30 Jul 2024 17:12:27 -0300 Subject: [PATCH] introduce pkg structure (#9) --- main.go | 19 ++++++++++--------- main_test.go | 6 ++++-- db.go => ngtop/db.go | 18 +++++++++--------- fields.go => ngtop/fields.go | 2 +- parser.go => ngtop/parser.go | 2 +- parser_test.go => ngtop/parser_test.go | 12 +++++++++++- 6 files changed, 36 insertions(+), 23 deletions(-) rename db.go => ngtop/db.go (92%) rename fields.go => ngtop/fields.go (99%) rename parser.go => ngtop/parser.go (99%) rename parser_test.go => ngtop/parser_test.go (89%) diff --git a/main.go b/main.go index 0351106..d036000 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "time" "github.com/alecthomas/kong" + "github.com/facundoolano/ngtop/ngtop" ) type CommandArgs struct { @@ -54,10 +55,10 @@ func main() { logFormat = envLogFormat } - parser := NewParser(logFormat) + parser := ngtop.NewParser(logFormat) ctx, spec := querySpecFromCLI() - dbs, err := InitDB(dbPath, parser.Fields) + dbs, err := ngtop.InitDB(dbPath, parser.Fields) ctx.FatalIfErrorf(err) defer dbs.Close() @@ -70,10 +71,10 @@ func main() { } // Parse the command line arguments into a top requests query specification -func querySpecFromCLI() (*kong.Context, *RequestCountSpec) { +func querySpecFromCLI() (*kong.Context, *ngtop.RequestCountSpec) { // Parse query spec first, i.e. don't bother with db updates if the command is invalid - fieldNames := make([]string, 0, len(CLI_NAME_TO_FIELD)) - for k := range CLI_NAME_TO_FIELD { + fieldNames := make([]string, 0, len(ngtop.CLI_NAME_TO_FIELD)) + for k := range ngtop.CLI_NAME_TO_FIELD { fieldNames = append(fieldNames, k) } @@ -96,13 +97,13 @@ func querySpecFromCLI() (*kong.Context, *RequestCountSpec) { // translate field name aliases columns := make([]string, len(cli.Fields)) for i, field := range cli.Fields { - columns[i] = CLI_NAME_TO_FIELD[field].ColumnName + columns[i] = ngtop.CLI_NAME_TO_FIELD[field].ColumnName } whereConditions, err := resolveWhereConditions(cli.Where) ctx.FatalIfErrorf(err) - spec := &RequestCountSpec{ + spec := &ngtop.RequestCountSpec{ GroupByMetrics: columns, TimeSince: since, TimeUntil: until, @@ -129,7 +130,7 @@ func resolveWhereConditions(clauses []string) (map[string][]string, error) { return nil, fmt.Errorf("invalid where expression %s", clause) } - if field, found := CLI_NAME_TO_FIELD[keyvalue[0]]; found { + if field, found := ngtop.CLI_NAME_TO_FIELD[keyvalue[0]]; found { conditions[field.ColumnName] = append(conditions[field.ColumnName], keyvalue[1]) } else { return nil, fmt.Errorf("unknown field name %s", keyvalue[0]) @@ -172,7 +173,7 @@ func parseDuration(duration string) (time.Time, error) { } // Parse the most recent nginx access.logs and insert the ones not previously seen into the DB. -func loadLogs(parser *LogParser, logPathPattern string, dbs *dbSession) error { +func loadLogs(parser *ngtop.LogParser, logPathPattern string, dbs *ngtop.DBSession) error { logFiles, err := filepath.Glob(logPathPattern) if err != nil { return err diff --git a/main_test.go b/main_test.go index 595b235..57b53d0 100644 --- a/main_test.go +++ b/main_test.go @@ -5,6 +5,8 @@ import ( "reflect" "testing" "time" + + "github.com/facundoolano/ngtop/ngtop" ) func TestDurationParsing(t *testing.T) { @@ -286,8 +288,8 @@ func runCommand(t *testing.T, format string, logs string, cliArgs []string) ([]s os.Args = append([]string{"ngtop"}, cliArgs...) _, spec := querySpecFromCLI() - parser := NewParser(format) - dbs, err := InitDB(dbFile.Name(), parser.Fields) + parser := ngtop.NewParser(format) + dbs, err := ngtop.InitDB(dbFile.Name(), parser.Fields) assertEqual(t, err, nil) defer dbs.Close() diff --git a/db.go b/ngtop/db.go similarity index 92% rename from db.go rename to ngtop/db.go index fc4bc37..c3e33fe 100644 --- a/db.go +++ b/ngtop/db.go @@ -1,4 +1,4 @@ -package main +package ngtop import ( "database/sql" @@ -18,7 +18,7 @@ type RequestCountSpec struct { Where map[string][]string } -type dbSession struct { +type DBSession struct { db *sql.DB columns []string insertTx *sql.Tx @@ -28,7 +28,7 @@ type dbSession struct { const DB_DATE_LAYOUT = "2006-01-02 15:04:05-07:00" // Open or create the database at the given path. -func InitDB(dbPath string, fields []*LogField) (*dbSession, error) { +func InitDB(dbPath string, fields []*LogField) (*DBSession, error) { db, err := sql.Open("sqlite3", dbPath) if err != nil { return nil, err @@ -55,15 +55,15 @@ func InitDB(dbPath string, fields []*LogField) (*dbSession, error) { );`, columnSpecs) _, err = db.Exec(sqlStmt) - return &dbSession{db: db, columns: columns}, err + return &DBSession{db: db, columns: columns}, err } -func (dbs *dbSession) Close() { +func (dbs *DBSession) Close() { dbs.db.Close() } // Prepare a transaction to insert a new batch of log entries, returning the time of the last seen log entry. -func (dbs *dbSession) PrepareForUpdate() (*time.Time, error) { +func (dbs *DBSession) PrepareForUpdate() (*time.Time, error) { // we want to avoid processed files that were already processed in the past. but we still want to add new log entries // from the most recent files, which may have been extended since we last saw them. // Since there is no "uniqueness" in logs (even the same ip can make the same request at the same second ---I checked), @@ -99,14 +99,14 @@ func (dbs *dbSession) PrepareForUpdate() (*time.Time, error) { return lastSeemTime, nil } -func (dbs *dbSession) AddLogEntry(values []any) error { +func (dbs *DBSession) AddLogEntry(values []any) error { _, err := dbs.insertStmt.Exec(values...) return err } // If the given processing `err` is nil, commit the log insertion transaction, // Otherwise roll it back and return the error. -func (dbs *dbSession) FinishUpdate(err error) error { +func (dbs *DBSession) FinishUpdate(err error) error { tx := dbs.insertTx dbs.insertTx = nil dbs.insertStmt = nil @@ -118,7 +118,7 @@ func (dbs *dbSession) FinishUpdate(err error) error { } // Build a query from the spec and execute it, returning the results as stringified values. -func (dbs *dbSession) QueryTop(spec *RequestCountSpec) ([]string, [][]string, error) { +func (dbs *DBSession) QueryTop(spec *RequestCountSpec) ([]string, [][]string, error) { queryString, queryArgs := spec.buildQuery() rows, err := dbs.db.Query(queryString, queryArgs...) diff --git a/fields.go b/ngtop/fields.go similarity index 99% rename from fields.go rename to ngtop/fields.go index f3b59a9..56eb354 100644 --- a/fields.go +++ b/ngtop/fields.go @@ -1,4 +1,4 @@ -package main +package ngtop import ( "github.com/mileusna/useragent" diff --git a/parser.go b/ngtop/parser.go similarity index 99% rename from parser.go rename to ngtop/parser.go index 7fdc1cb..1a48f6a 100644 --- a/parser.go +++ b/ngtop/parser.go @@ -1,4 +1,4 @@ -package main +package ngtop import ( "bufio" diff --git a/parser_test.go b/ngtop/parser_test.go similarity index 89% rename from parser_test.go rename to ngtop/parser_test.go index 953fee8..9f84c27 100644 --- a/parser_test.go +++ b/ngtop/parser_test.go @@ -1,9 +1,12 @@ -package main +package ngtop import ( + "reflect" "testing" ) +const DEFAULT_LOG_FORMAT = `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"` + func TestFormatRegex(t *testing.T) { line := `xx.xx.xx.xx - - [24/Jul/2024:00:00:28 +0000] "GET /feed HTTP/1.1" 301 169 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"` @@ -52,3 +55,10 @@ func TestRefererOverride(t *testing.T) { assertEqual(t, err, nil) assertEqual(t, result["referer"], "olano.dev/feed.xml") } + +func assertEqual(t *testing.T, a interface{}, b interface{}) { + t.Helper() + if !reflect.DeepEqual(a, b) { + t.Fatalf("%v != %v", a, b) + } +}