FS-1053 Fix firmware record updates for existing firmware (#154)

* FS-1053 Fix firmware record updates for existing firmware

Makefile

@@ -10,6 +10,11 @@ REPO := "https://github.com/metal-toolbox/firmware-syncer.git"
 
 .DEFAULT_GOAL := help
 
+## Generate mocks
+mocks:
+	go get go.uber.org/mock@v0.3.0
+	go install go.uber.org/mock/mockgen@v0.3.0
+	go generate ./...
 
 ## Go test
 test:

cmd/root.go

@@ -30,17 +30,16 @@ var rootCmd = &cobra.Command{
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		syncerApp.Logger.Info("Sync starting")
 		err = syncerApp.SyncFirmwares(cmd.Context())
 		if err != nil {
 			syncerApp.Logger.Fatal(err)
 		}
+		syncerApp.Logger.Info("Sync complete")
 	},
 }
 
-func NewRootCmd() *cobra.Command {
-	return rootCmd
-}
-
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/google/go-github/v53 v53.2.0
-	github.com/google/go-github/v56 v56.0.0
 	github.com/google/uuid v1.4.0
 	github.com/jeremywohl/flatten v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0
@@ -23,7 +22,7 @@ require (
 	github.com/spf13/viper v1.17.0
 	github.com/stretchr/testify v1.8.4
 	go.hollow.sh/serverservice v0.16.2
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+	go.uber.org/mock v0.3.0
 	golang.org/x/net v0.18.0
 	golang.org/x/oauth2 v0.14.0
 )
@@ -136,7 +135,8 @@ require (
 	gocloud.dev v0.33.0 // indirect
 	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/crypto v0.15.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/term v0.14.0 // indirect
 	golang.org/x/text v0.14.0 // indirect

internal/app/app.go

@@ -13,18 +13,19 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 
-	runtime "github.com/banzaicloud/logrus-runtime-formatter"
 	"github.com/metal-toolbox/firmware-syncer/internal/config"
+	"github.com/metal-toolbox/firmware-syncer/internal/inventory"
+	"github.com/metal-toolbox/firmware-syncer/internal/logging"
 	"github.com/metal-toolbox/firmware-syncer/internal/vendors"
-	"github.com/metal-toolbox/firmware-syncer/internal/vendors/asrockrack"
-	"github.com/metal-toolbox/firmware-syncer/internal/vendors/dell"
-	"github.com/metal-toolbox/firmware-syncer/internal/vendors/equinix"
-	"github.com/metal-toolbox/firmware-syncer/internal/vendors/intel"
-	"github.com/metal-toolbox/firmware-syncer/internal/vendors/mellanox"
+	"github.com/metal-toolbox/firmware-syncer/internal/vendors/github"
 	"github.com/metal-toolbox/firmware-syncer/internal/vendors/supermicro"
 	"github.com/metal-toolbox/firmware-syncer/pkg/types"
 )
 
+const (
+	VendorEquinix = "equinix"
+)
+
 // App holds attributes for the firmware-syncer application
 type App struct {
 	// Viper loads configuration parameters.
@@ -42,8 +43,8 @@ func New(ctx context.Context, inventoryKind types.InventoryKind, cfgFile, logLev
 	app := &App{
 		v:      viper.New(),
 		Config: &config.Configuration{},
-		Logger: logrus.New(),
 	}
+
 	if err := app.LoadConfiguration(cfgFile, inventoryKind); err != nil {
 		return nil, err
 	}
@@ -53,97 +54,59 @@ func New(ctx context.Context, inventoryKind types.InventoryKind, cfgFile, logLev
 		app.Config.LogLevel = logLevel
 	}
 
-	switch types.LogLevel(app.Config.LogLevel) {
-	case types.LogLevelDebug:
-		app.Logger.Level = logrus.DebugLevel
-	case types.LogLevelTrace:
-		app.Logger.Level = logrus.TraceLevel
-	default:
-		app.Logger.Level = logrus.InfoLevel
+	app.Logger = logging.NewLogger(app.Config.LogLevel)
+
+	// Load firmware manifest
+	firmwaresByVendor, err := config.LoadFirmwareManifest(ctx, app.Config.FirmwareManifestURL)
+	if err != nil {
+		app.Logger.Error(err.Error())
+		return nil, err
 	}
 
-	runtimeFormatter := &runtime.Formatter{
-		ChildFormatter: &logrus.JSONFormatter{},
-		File:           true,
-		Line:           true,
-		BaseNameOnly:   true,
+	inventoryClient, err := inventory.New(ctx, app.Config.ServerserviceOptions, app.Config.ArtifactsURL, app.Logger)
+	if err != nil {
+		return nil, err
 	}
 
-	app.Logger.SetFormatter(runtimeFormatter)
+	dstFs, err := vendors.InitS3Fs(ctx, app.Config.FirmwareRepository, "/")
+	if err != nil {
+		return nil, err
+	}
 
-	// Load firmware manifest
-	firmwaresByVendor, err := config.LoadFirmwareManifest(ctx, app.Config.FirmwareManifestURL)
+	tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: os.TempDir()})
 	if err != nil {
-		app.Logger.Error(err.Error())
 		return nil, err
 	}
 
 	for vendor, firmwares := range firmwaresByVendor {
+		var downloader vendors.Downloader
+
 		switch vendor {
 		case common.VendorDell:
-			var dup vendors.Vendor
-
-			dup, err = dell.NewDUP(ctx, firmwares, app.Config, app.Logger)
-			if err != nil {
-				app.Logger.Error("Failed to initialize Dell vendor: " + err.Error())
-				return nil, err
-			}
-
-			app.vendors = append(app.vendors, dup)
+			downloader = vendors.NewRcloneDownloader(app.Logger)
 		case common.VendorAsrockrack:
-			var asrr vendors.Vendor
-
-			asrr, err = asrockrack.New(ctx, firmwares, app.Config, app.Logger)
+			s3Fs, err := vendors.InitS3Fs(ctx, app.Config.AsRockRackRepository, "/")
 			if err != nil {
-				app.Logger.Error("Failed to initialize ASRockRack vendor:" + err.Error())
 				return nil, err
 			}
 
-			app.vendors = append(app.vendors, asrr)
+			downloader = vendors.NewS3Downloader(app.Logger, s3Fs)
 		case common.VendorSupermicro:
-			var sm vendors.Vendor
-
-			sm, err = supermicro.New(ctx, firmwares, app.Config, app.Logger)
-			if err != nil {
-				app.Logger.Error("Failed to initialize Supermicro vendor: " + err.Error())
-				return nil, err
-			}
-
-			app.vendors = append(app.vendors, sm)
+			downloader = supermicro.NewSupermicroDownloader(app.Logger)
 		case common.VendorMellanox:
-			var mlx vendors.Vendor
-
-			mlx, err = mellanox.New(ctx, firmwares, app.Config, app.Logger)
-			if err != nil {
-				app.Logger.Error("Failed to initialize Mellanox vendor: " + err.Error())
-				return nil, err
-			}
-
-			app.vendors = append(app.vendors, mlx)
+			downloader = vendors.NewArchiveDownloader(app.Logger)
 		case common.VendorIntel:
-			var i vendors.Vendor
-
-			i, err = intel.New(ctx, firmwares, app.Config, app.Logger)
-			if err != nil {
-				app.Logger.Error("Failed to initialize Intel vendor: " + err.Error())
-				return nil, err
-			}
-
-			app.vendors = append(app.vendors, i)
-		case "equinix":
-			var e vendors.Vendor
-
-			e, err = equinix.New(ctx, firmwares, app.Config, app.Logger)
-			if err != nil {
-				app.Logger.Error("Failed to initialize Equinix vendor: " + err.Error())
-				return nil, err
-			}
-
-			app.vendors = append(app.vendors, e)
+			downloader = vendors.NewArchiveDownloader(app.Logger)
+		case VendorEquinix:
+			ghClient := github.NewGitHubClient(ctx, app.Config.GithubOpenBmcToken)
+			downloader = github.NewGitHubDownloader(app.Logger, ghClient)
 		default:
 			app.Logger.Error("Vendor not supported: " + vendor)
 			continue
 		}
+
+		syncer := vendors.NewSyncer(dstFs, tmpFs, downloader, inventoryClient, firmwares, app.Logger)
+		app.vendors = append(app.vendors, syncer)
 	}
 
 	return app, nil
@@ -154,7 +117,7 @@ func (a *App) SyncFirmwares(ctx context.Context) error {
 	for _, v := range a.vendors {
 		err := v.Sync(ctx)
 		if err != nil {
-			a.Logger.Error("Failed to sync: " + err.Error())
+			a.Logger.WithError(err).Error("Failed to sync vendor")
 		}
 	}