From d9f32fbf22bdc3a770deb7ffe6c143fe42229eaa Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Thu, 3 Feb 2022 09:55:56 +0100 Subject: [PATCH] Add command to check for missing data (#29) * Add check_missing command to query for missing entries. * Fix exit handler with expected exits --- check_command.go | 72 ++++++++++++++++++++++++++++++++++ main.go | 16 ++++++-- migrate_command.go | 3 +- pkg/check/missing.go | 38 ++++++++++++++++++ pkg/check/missing_test.go | 82 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 check_command.go create mode 100644 pkg/check/missing.go create mode 100644 pkg/check/missing_test.go diff --git a/check_command.go b/check_command.go new file mode 100644 index 0000000..40dbe1a --- /dev/null +++ b/check_command.go @@ -0,0 +1,72 @@ +package main + +import ( + "database/sql" + "fmt" + "os" + "text/tabwriter" + + "github.com/urfave/cli/v2" + + "github.com/appuio/appuio-cloud-reporting/pkg/check" + "github.com/appuio/appuio-cloud-reporting/pkg/db" +) + +type checkMissingCommand struct { + DatabaseURL string +} + +var checkMissingCommandName = "check_missing" + +func newCheckMissingCommand() *cli.Command { + command := &checkMissingCommand{} + return &cli.Command{ + Name: checkMissingCommandName, + Usage: "Check for missing data in the database", + Before: command.before, + Action: command.execute, + Flags: []cli.Flag{ + newDbURLFlag(&command.DatabaseURL), + }, + } +} + +func (cmd *checkMissingCommand) before(context *cli.Context) error { + return LogMetadata(context) +} + +func (cmd *checkMissingCommand) execute(cliCtx *cli.Context) error { + ctx := cliCtx.Context + log := AppLogger(ctx).WithName(migrateCommandName) + + log.V(1).Info("Opening database connection", "url", cmd.DatabaseURL) + rdb, err := db.Openx(cmd.DatabaseURL) + if err != nil { + return fmt.Errorf("could not open database connection: %w", err) + } + + log.V(1).Info("Begin transaction") + tx, err := rdb.BeginTxx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer tx.Rollback() + + missing, err := check.Missing(ctx, tx) + if err != nil { + return err + } + + if len(missing) == 0 { + return nil + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + defer w.Flush() + fmt.Fprint(w, "Table\tMissing Field\tID\tSource\n") + for _, m := range missing { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", m.Table, m.MissingField, m.ID, m.Source) + } + + return cli.Exit(fmt.Sprintf("%d missing entries found.", len(missing)), 1) +} diff --git a/main.go b/main.go index d490113..34cd1a4 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -66,14 +67,21 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) { Commands: []*cli.Command{ newMigrateCommand(), newReportCommand(), + newCheckMissingCommand(), newInvoiceCommand(), }, ExitErrHandler: func(context *cli.Context, err error) { - if err != nil { - AppLogger(context.Context).WithCallDepth(1).Error(err, "fatal error") - os.Exit(1) - cli.HandleExitCoder(cli.Exit("", 1)) + if err == nil { + return } + // Don't show stack trace if the error is expected (someone called cli.Exit()) + var exitErr cli.ExitCoder + if errors.As(err, &exitErr) { + cli.HandleExitCoder(err) + return + } + AppLogger(context.Context).WithCallDepth(1).Error(err, "fatal error") + cli.OsExiter(1) }, } // There is logr.NewContext(...) which returns a context that carries the logger instance. diff --git a/migrate_command.go b/migrate_command.go index 0e6e997..56414d0 100644 --- a/migrate_command.go +++ b/migrate_command.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "github.com/appuio/appuio-cloud-reporting/pkg/db" "github.com/urfave/cli/v2" @@ -66,7 +65,7 @@ func (cmd *migrateCommand) execute(context *cli.Context) error { // non-zero exit code could be used in scripts if len(pm) > 0 { - os.Exit(1) + cli.Exit("Pending migrations found.", 1) } return nil } diff --git a/pkg/check/missing.go b/pkg/check/missing.go new file mode 100644 index 0000000..7e04d43 --- /dev/null +++ b/pkg/check/missing.go @@ -0,0 +1,38 @@ +package check + +import ( + "context" + "fmt" + + "github.com/jmoiron/sqlx" +) + +// MissingField represents a missing field. +type MissingField struct { + Table string + + ID string + Source string + + MissingField string +} + +const missingQuery = ` + SELECT 'categories' as table, id, source, 'target' as missingfield FROM categories WHERE target IS NULL OR target = '' + UNION ALL + SELECT 'tenants' as table, id, source, 'target' as missingfield FROM tenants WHERE target IS NULL OR target = '' + UNION ALL + SELECT 'products' as table, id, source, 'target' as missingfield FROM products WHERE target IS NULL OR target = '' + UNION ALL + SELECT 'products' as table, id, source, 'amount' as missingfield FROM products WHERE amount = 0 + UNION ALL + SELECT 'products' as table, id, source, 'unit' as missingfield FROM products WHERE unit = '' +` + +// Missing checks for missing fields in the reporting database. +func Missing(ctx context.Context, tx sqlx.QueryerContext) ([]MissingField, error) { + var missing []MissingField + + err := sqlx.SelectContext(ctx, tx, &missing, fmt.Sprintf(`WITH missing AS (%s) SELECT * FROM missing ORDER BY "table",missingfield,source`, missingQuery)) + return missing, err +} diff --git a/pkg/check/missing_test.go b/pkg/check/missing_test.go new file mode 100644 index 0000000..84ea48b --- /dev/null +++ b/pkg/check/missing_test.go @@ -0,0 +1,82 @@ +package check_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/appuio/appuio-cloud-reporting/pkg/check" + "github.com/appuio/appuio-cloud-reporting/pkg/db" + "github.com/appuio/appuio-cloud-reporting/pkg/db/dbtest" +) + +type TestSuite struct { + dbtest.Suite +} + +func (s *TestSuite) TestMissingFields() { + t := s.T() + tx := s.Begin() + defer tx.Rollback() + + m, err := check.Missing(context.Background(), tx) + require.NoError(t, err) + require.Len(t, m, 0) + + expectedMissing := s.requireMissingTestEntries(t, tx) + + m, err = check.Missing(context.Background(), tx) + require.NoError(t, err) + require.Equal(t, expectedMissing, m) +} + +func (s *TestSuite) requireMissingTestEntries(t *testing.T, tdb *sqlx.Tx) []check.MissingField { + var catEmptyTarget db.Category + require.NoError(t, + db.GetNamed(tdb, &catEmptyTarget, + "INSERT INTO categories (source,target) VALUES (:source,:target) RETURNING *", db.Category{ + Source: "af-south-1:uroboros-research", + })) + + var tenantEmptyTarget db.Tenant + require.NoError(t, + db.GetNamed(tdb, &tenantEmptyTarget, + "INSERT INTO tenants (source,target) VALUES (:source,:target) RETURNING *", db.Tenant{ + Source: "tricell", + })) + + var productEmptyTarget db.Product + require.NoError(t, + db.GetNamed(tdb, &productEmptyTarget, + "INSERT INTO products (source,target,amount,unit,during) VALUES (:source,:target,:amount,:unit,:during) RETURNING *", db.Product{ + Source: "test_memory:us-rac-2", + Amount: 3, + Unit: "X", + During: db.InfiniteRange(), + })) + + var productEmptyAmountAndUnit db.Product + require.NoError(t, + db.GetNamed(tdb, &productEmptyAmountAndUnit, + "INSERT INTO products (source,target,amount,unit,during) VALUES (:source,:target,:amount,:unit,:during) RETURNING *", db.Product{ + Source: "test_storage:us-rac-2", + Target: sql.NullString{Valid: true, String: "666"}, + During: db.InfiniteRange(), + })) + + return []check.MissingField{ + {Table: "categories", MissingField: "target", ID: catEmptyTarget.Id, Source: catEmptyTarget.Source}, + {Table: "products", MissingField: "amount", ID: productEmptyAmountAndUnit.Id, Source: productEmptyAmountAndUnit.Source}, + {Table: "products", MissingField: "target", ID: productEmptyTarget.Id, Source: productEmptyTarget.Source}, + {Table: "products", MissingField: "unit", ID: productEmptyAmountAndUnit.Id, Source: productEmptyAmountAndUnit.Source}, + {Table: "tenants", MissingField: "target", ID: tenantEmptyTarget.Id, Source: tenantEmptyTarget.Source}, + } +} + +func TestTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +}