Move env configuration loading back to main (#4)

* handle environment configuration in main

* adjust tests

* constant for default db path

accesslog.go

@@ -15,10 +15,6 @@ import (
 	"time"
 )
 
-// defaulting to the default Debian location (and presumably other linuxes)
-// overridable with NGTOP_LOGS_PATH env var
-const DEFAULT_PATH = "/var/log/nginx/access.log*"
-
 // TODO add support to nginx config syntax, eg "$remote_addr - $remote_user [$time_local] ..." and add code to translate it to these regexes
 // FIXME consolidate field list (duplicated knowledge)
 const LOG_COMBINED_PATTERN = `(?P<ip>\S+) - (?P<remote_user>\S+) \[(?P<time>.*?)\] "(?P<request_raw>[^"]*)" (?P<status>\d{3}) (?P<bytes_sent>\d+) "(?P<referer>[^"]*)" "(?P<user_agent_raw>[^"]*)"`
@@ -27,18 +23,7 @@ var logPattern = regexp.MustCompile(LOG_COMBINED_PATTERN)
 
 // Parse the fields in the nginx access logs since the `until` time, passing them as a map into the `processFun`.
 // Processing is interrupted when a log older than `until` is found.
-func ProcessAccessLogs(until *time.Time, processFun func(map[string]interface{}) error) error {
-
-	// could make sense to try detecting the OS and applying a sensible default accordingly
-	accessLogsPath := DEFAULT_PATH
-	if envLogsPath := os.Getenv("NGTOP_LOGS_PATH"); envLogsPath != "" {
-		accessLogsPath = envLogsPath
-	}
-	logFiles, err := filepath.Glob(accessLogsPath)
-	if err != nil {
-		return err
-	}
-
+func ProcessAccessLogs(logFiles []string, until *time.Time, processFun func(map[string]interface{}) error) error {
 	for _, path := range logFiles {
 
 		log.Printf("parsing %s", path)

db.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	_ "github.com/mattn/go-sqlite3"
 	"log"
-	"os"
 	"strings"
 	"time"
 )
@@ -25,12 +24,8 @@ type dbSession struct {
 	insertStmt *sql.Stmt
 }
 
-// TODO doc
-func InitDB() (*dbSession, error) {
-	dbPath := "./ngtop.db"
-	if envPath := os.Getenv("NGTOP_DB"); envPath != "" {
-		dbPath = envPath
-	}
+// Open or create the database at the given path.
+func InitDB(dbPath string) (*dbSession, error) {
 	db, err := sql.Open("sqlite3", dbPath)
 	if err != nil {
 		return nil, err

main.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"math"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -49,19 +50,33 @@ var FIELD_NAMES = map[string]string{
 // Use a var to get current time, allowing for tests to override it
 var NowTimeFun = time.Now
 
+// defaulting to the default Debian location (and presumably other linuxes)
+// overridable with NGTOP_LOGS_PATH env var
+const DEFAULT_PATH_PATTERN = "/var/log/nginx/access.log*"
+const DEFAULT_DB_PATH = "./ngtop.db"
+
 func main() {
 	// Optionally enable internal logger
 	if os.Getenv("NGTOP_LOG") == "" {
 		log.Default().SetOutput(io.Discard)
 	}
 
-	ctx, spec := querySpecFromCLI()
+	dbPath := DEFAULT_DB_PATH
+	if envPath := os.Getenv("NGTOP_DB"); envPath != "" {
+		dbPath = envPath
+	}
+
+	logPathPattern := DEFAULT_PATH_PATTERN
+	if envLogsPath := os.Getenv("NGTOP_LOGS_PATH"); envLogsPath != "" {
+		logPathPattern = envLogsPath
+	}
 
-	dbs, err := InitDB()
+	ctx, spec := querySpecFromCLI()
+	dbs, err := InitDB(dbPath)
 	ctx.FatalIfErrorf(err)
 	defer dbs.Close()
 
-	err = loadLogs(dbs)
+	err = loadLogs(logPathPattern, dbs)
 	ctx.FatalIfErrorf(err)
 
 	columnNames, rowValues, err := dbs.QueryTop(spec)
@@ -172,7 +187,12 @@ func parseDuration(duration string) (time.Time, error) {
 }
 
 // Parse the most recent nginx access.logs and insert the ones not previously seen into the DB.
-func loadLogs(dbs *dbSession) error {
+func loadLogs(logPathPattern string, dbs *dbSession) error {
+	logFiles, err := filepath.Glob(logPathPattern)
+	if err != nil {
+		return err
+	}
+
 	// FIXME consolidate field list (duplicated knowledge)
 	dbColumns := []string{"ip", "time", "request_raw", "status", "bytes_sent", "referer", "user_agent_raw", "method", "path", "user_agent", "os", "device", "ua_url", "ua_type"}
 
@@ -181,7 +201,7 @@ func loadLogs(dbs *dbSession) error {
 		return err
 	}
 
-	err = ProcessAccessLogs(lastSeenTime, func(logLineFields map[string]interface{}) error {
+	err = ProcessAccessLogs(logFiles, lastSeenTime, func(logLineFields map[string]interface{}) error {
 		queryValues := make([]interface{}, len(dbColumns))
 		for i, field := range dbColumns {
 			queryValues[i] = logLineFields[field]

main_test.go

@@ -251,28 +251,25 @@ func TestMultipleLogFiles(t *testing.T) {
 
 func runCommand(t *testing.T, logs string, cliArgs []string) ([]string, [][]string) {
 	// write the logs to a temp file, and point the NGTOP_LOGS_PATH env to it
-	f, err := os.CreateTemp("", "access.log")
+	logFile, err := os.CreateTemp("", "access.log")
 	assertEqual(t, err, nil)
-	defer os.Remove(f.Name())
-	_, err = f.Write([]byte(logs))
+	defer os.Remove(logFile.Name())
+	_, err = logFile.Write([]byte(logs))
 	assertEqual(t, err, nil)
-	t.Setenv("NGTOP_LOGS_PATH", f.Name())
 
 	// create a one-off db file for the test
-	f, err = os.CreateTemp("", "ngtop.db")
+	dbFile, err := os.CreateTemp("", "ngtop.db")
 	assertEqual(t, err, nil)
-	defer os.Remove(f.Name())
-	os.Setenv("NGTOP_DB", f.Name())
+	defer os.Remove(dbFile.Name())
 
-	// some duplication from main here, maybe can refactored away
 	os.Args = append([]string{"ngtop"}, cliArgs...)
 	_, spec := querySpecFromCLI()
 
-	dbs, err := InitDB()
+	dbs, err := InitDB(dbFile.Name())
 	assertEqual(t, err, nil)
 	defer dbs.Close()
 
-	err = loadLogs(dbs)
+	err = loadLogs(logFile.Name(), dbs)
 	assertEqual(t, err, nil)
 	columnNames, rowValues, err := dbs.QueryTop(spec)
 	assertEqual(t, err, nil)