From 389ac6fa60b79cb24706de0d40813b0f4eb10e42 Mon Sep 17 00:00:00 2001 From: John Mears <20019566+coffeefreak101@users.noreply.github.com> Date: Tue, 19 Dec 2023 10:52:22 -0700 Subject: [PATCH] FS-1053 Fix firmware record updates for existing firmware (#154) * FS-1053 Fix firmware record updates for existing firmware --- Makefile | 5 + cmd/root.go | 7 +- go.mod | 6 +- go.sum | 68 ++- internal/app/app.go | 113 ++-- internal/inventory/mocks/serverservice.go | 54 ++ internal/inventory/serverservice.go | 184 ++++--- internal/inventory/serverservice_test.go | 1 + internal/logging/logger.go | 33 ++ internal/vendors/asrockrack/asrockrack.go | 126 ----- internal/vendors/dell/dell.go | 138 ----- internal/vendors/downloader.go | 155 +++--- internal/vendors/equinix/equinix.go | 186 ------- internal/vendors/fixtures/foobar.zip | Bin 0 -> 177 bytes internal/vendors/github/github.go | 117 ++++ .../equinix_test.go => github/github_test.go} | 2 +- internal/vendors/intel/intel.go | 136 ----- internal/vendors/mellanox/mellanox.go | 131 ----- internal/vendors/mocks/downloader.go | 55 ++ internal/vendors/mocks/rclone.go | 501 ++++++++++++++++++ internal/vendors/supermicro/supermicro.go | 161 ++---- internal/vendors/syncer.go | 124 +++++ internal/vendors/syncer_test.go | 154 ++++++ internal/vendors/vendors.go | 1 - 24 files changed, 1351 insertions(+), 1107 deletions(-) create mode 100644 internal/inventory/mocks/serverservice.go create mode 100644 internal/logging/logger.go delete mode 100644 internal/vendors/asrockrack/asrockrack.go delete mode 100644 internal/vendors/dell/dell.go delete mode 100644 internal/vendors/equinix/equinix.go create mode 100644 internal/vendors/fixtures/foobar.zip create mode 100644 internal/vendors/github/github.go rename internal/vendors/{equinix/equinix_test.go => github/github_test.go} (98%) delete mode 100644 internal/vendors/intel/intel.go delete mode 100644 internal/vendors/mellanox/mellanox.go create mode 100644 internal/vendors/mocks/downloader.go create mode 100644 internal/vendors/mocks/rclone.go create mode 100644 internal/vendors/syncer.go create mode 100644 internal/vendors/syncer_test.go diff --git a/Makefile b/Makefile index 57bea5c9..483666e9 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,11 @@ REPO := "https://github.com/metal-toolbox/firmware-syncer.git" .DEFAULT_GOAL := help +## Generate mocks +mocks: + go get go.uber.org/mock@v0.3.0 + go install go.uber.org/mock/mockgen@v0.3.0 + go generate ./... ## Go test test: diff --git a/cmd/root.go b/cmd/root.go index 9a879fe1..17718016 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,17 +30,16 @@ var rootCmd = &cobra.Command{ if err != nil { log.Fatal(err) } + + syncerApp.Logger.Info("Sync starting") err = syncerApp.SyncFirmwares(cmd.Context()) if err != nil { syncerApp.Logger.Fatal(err) } + syncerApp.Logger.Info("Sync complete") }, } -func NewRootCmd() *cobra.Command { - return rootCmd -} - // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { diff --git a/go.mod b/go.mod index da2d4a0c..6ebf7cd6 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a github.com/coreos/go-oidc v2.2.1+incompatible github.com/google/go-github/v53 v53.2.0 - github.com/google/go-github/v56 v56.0.0 github.com/google/uuid v1.4.0 github.com/jeremywohl/flatten v1.0.1 github.com/mitchellh/mapstructure v1.5.0 @@ -23,7 +22,7 @@ require ( github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 go.hollow.sh/serverservice v0.16.2 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + go.uber.org/mock v0.3.0 golang.org/x/net v0.18.0 golang.org/x/oauth2 v0.14.0 ) @@ -136,7 +135,8 @@ require ( gocloud.dev v0.33.0 // indirect golang.org/x/arch v0.4.0 // indirect golang.org/x/crypto v0.15.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/term v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 1a3b93ae..c22f3aa5 100644 --- a/go.sum +++ b/go.sum @@ -81,10 +81,14 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/gopenpgp/v2 v2.7.3 h1:AJu1OI/1UWVYZl6QcCLKGu9OTngS2r52618uGlje84I= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= @@ -93,6 +97,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apmckinlay/gsuneido v0.0.0-20190404155041-0b6cd442a18f/go.mod h1:JU2DOj5Fc6rol0yaT79Csr47QR0vONGwJtBNGRD7jmc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -112,8 +117,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a h1:SjtoU9dE3bYfYnPXODCunMztjoDgnE3DVJCPLBqwz6Q= github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a/go.mod h1:SY//n1PJjZfbFbmAsB6GvEKbc7UXz3d30s3kWxfJQ/c= +github.com/bradenaw/juniper v0.13.1 h1:9P7/xeaYuEyqPuJHSHCJoisWyPvZH4FAi59BxJLh7F8= github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= @@ -137,7 +142,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -153,7 +157,7 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go/v2 v2.3.5 h1:Khtm8K6fTTz/ZCWPzU9Ne3aOW9VyAnj4qIPCJgKtwK0= github.com/cockroachdb/cockroach-go/v2 v2.3.5/go.mod h1:1wNJ45eSXW9AnOc3skntW9ZUZz6gxrQK3cOj3rK+BC8= -github.com/colinmarc/hdfs/v2 v2.3.0 h1:tMxOjXn6+7iPUlxAyup9Ha2hnmLe3Sv5DM2qqbSQ2VY= +github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -166,6 +170,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -174,6 +179,9 @@ github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/emersion/go-message v0.17.0 h1:NIdSKHiVUx4qKqdd0HyJFD41cW8iFguM2XJnRZWQH04= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -190,6 +198,7 @@ github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05/go.mod h1:M9 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk= @@ -207,8 +216,6 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-contrib/zap v0.1.0 h1:RMSFFJo34XZogV62OgOzvrlaMNmXrNxmJ3bFmMwl6Cc= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= -github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -234,6 +241,7 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -312,7 +320,6 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= -github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -394,6 +401,8 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/henrybear327/Proton-API-Bridge v0.0.0-20230908065933-5bfa15b567db h1:EpttN0oeR6BvUPis9+4iSyxIacMHXlaHrD/+IvyKPlc= +github.com/henrybear327/go-proton-api v0.0.0-20230907193451-e563407504ce h1:n1URi7VYiwX/3akX51keQXi6Huy4lJdVc4biJHYk3iw= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -464,6 +473,7 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs= github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= +github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -480,6 +490,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU= +github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 h1:+A1uT26XjTsxiUUZjAAuveILWWy+Sy2TPX8OIgGvPQE= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= @@ -588,11 +599,9 @@ github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADym github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehdGZsm5LP7AhLOvjgbS8zN1tk= -github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= -github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/oracle/oci-go-sdk/v65 v65.34.0 h1:uG1KucBxAbn8cYRgQHxtQKogtl85nOX8LhimZCPfMqw= +github.com/oracle/oci-go-sdk/v65 v65.45.0 h1:EpCst/iZma9s8eYS0QJ9qsTmGxX5GPehYGN1jwGIteU= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -649,11 +658,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= -github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d h1:ZyH6ZfA/PzxF4qQS2MgFLXRdw/pWOSNJA7Lq0pkX49Y= -github.com/rclone/rclone v1.63.1 h1:iITCUNBfAXnguHjRPFq+w/gGIW0L0las78h4H5CH2Ms= -github.com/rclone/rclone v1.63.1/go.mod h1:eUQaKsf1wJfHKB0RDoM8RaPAeRB2eI/Qw+Vc9Ho5FGM= github.com/rclone/rclone v1.64.2 h1:vF1FiMw5xRpMKRdODme+EyCiit9spcyMkZcDmZXJG5M= github.com/rclone/rclone v1.64.2/go.mod h1:qvuTiatq811sVjaBRJ8RC4Z/CeZxIGdqahz778/YGJo= +github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= @@ -673,8 +680,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -695,7 +700,7 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spacemonkeygo/monkit/v3 v3.0.19 h1:wqBb9bpD7jXkVi4XwIp8jn1fektaVBQ+cp9SHRXgAdo= +github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 h1:QVUfVxilbPp8fBJ7701LL/WEUjBSiSxbs9LUaCIe5qM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= @@ -730,7 +735,6 @@ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= @@ -813,6 +817,8 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -854,8 +860,6 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -868,8 +872,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -953,8 +955,6 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -977,8 +977,6 @@ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -995,8 +993,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1092,10 +1090,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1105,8 +1100,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1121,8 +1114,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1459,6 +1450,7 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -storj.io/common v0.0.0-20221123115229-fed3e6651b63 h1:OuleF/3FvZe3Nnu6NdwVr+FvCXjfD4iNNdgfI2kcs3k= -storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= -storj.io/uplink v1.10.0 h1:3hS0hszupHSxEoC4DsMpljaRy0uNoijEPVF6siIE28Q= +storj.io/common v0.0.0-20230907123639-5fd0608fd947 h1:X75A5hX1nFjQH8GIvei4T1LNQTLa++bsDKMxXxfPHE8= +storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= +storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg= +storj.io/uplink v1.12.0 h1:rTODjbKRo/lzz5Hp0isjoRfqDcH7kJg6aujD2M9v9Ro= diff --git a/internal/app/app.go b/internal/app/app.go index 3d636fc4..cc0ddb90 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -13,18 +13,19 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/viper" - runtime "github.com/banzaicloud/logrus-runtime-formatter" "github.com/metal-toolbox/firmware-syncer/internal/config" + "github.com/metal-toolbox/firmware-syncer/internal/inventory" + "github.com/metal-toolbox/firmware-syncer/internal/logging" "github.com/metal-toolbox/firmware-syncer/internal/vendors" - "github.com/metal-toolbox/firmware-syncer/internal/vendors/asrockrack" - "github.com/metal-toolbox/firmware-syncer/internal/vendors/dell" - "github.com/metal-toolbox/firmware-syncer/internal/vendors/equinix" - "github.com/metal-toolbox/firmware-syncer/internal/vendors/intel" - "github.com/metal-toolbox/firmware-syncer/internal/vendors/mellanox" + "github.com/metal-toolbox/firmware-syncer/internal/vendors/github" "github.com/metal-toolbox/firmware-syncer/internal/vendors/supermicro" "github.com/metal-toolbox/firmware-syncer/pkg/types" ) +const ( + VendorEquinix = "equinix" +) + // App holds attributes for the firmware-syncer application type App struct { // Viper loads configuration parameters. @@ -42,8 +43,8 @@ func New(ctx context.Context, inventoryKind types.InventoryKind, cfgFile, logLev app := &App{ v: viper.New(), Config: &config.Configuration{}, - Logger: logrus.New(), } + if err := app.LoadConfiguration(cfgFile, inventoryKind); err != nil { return nil, err } @@ -53,97 +54,59 @@ func New(ctx context.Context, inventoryKind types.InventoryKind, cfgFile, logLev app.Config.LogLevel = logLevel } - switch types.LogLevel(app.Config.LogLevel) { - case types.LogLevelDebug: - app.Logger.Level = logrus.DebugLevel - case types.LogLevelTrace: - app.Logger.Level = logrus.TraceLevel - default: - app.Logger.Level = logrus.InfoLevel + app.Logger = logging.NewLogger(app.Config.LogLevel) + + // Load firmware manifest + firmwaresByVendor, err := config.LoadFirmwareManifest(ctx, app.Config.FirmwareManifestURL) + if err != nil { + app.Logger.Error(err.Error()) + return nil, err } - runtimeFormatter := &runtime.Formatter{ - ChildFormatter: &logrus.JSONFormatter{}, - File: true, - Line: true, - BaseNameOnly: true, + inventoryClient, err := inventory.New(ctx, app.Config.ServerserviceOptions, app.Config.ArtifactsURL, app.Logger) + if err != nil { + return nil, err } - app.Logger.SetFormatter(runtimeFormatter) + dstFs, err := vendors.InitS3Fs(ctx, app.Config.FirmwareRepository, "/") + if err != nil { + return nil, err + } - // Load firmware manifest - firmwaresByVendor, err := config.LoadFirmwareManifest(ctx, app.Config.FirmwareManifestURL) + tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: os.TempDir()}) if err != nil { - app.Logger.Error(err.Error()) return nil, err } for vendor, firmwares := range firmwaresByVendor { + var downloader vendors.Downloader + switch vendor { case common.VendorDell: - var dup vendors.Vendor - - dup, err = dell.NewDUP(ctx, firmwares, app.Config, app.Logger) - if err != nil { - app.Logger.Error("Failed to initialize Dell vendor: " + err.Error()) - return nil, err - } - - app.vendors = append(app.vendors, dup) + downloader = vendors.NewRcloneDownloader(app.Logger) case common.VendorAsrockrack: - var asrr vendors.Vendor - - asrr, err = asrockrack.New(ctx, firmwares, app.Config, app.Logger) + s3Fs, err := vendors.InitS3Fs(ctx, app.Config.AsRockRackRepository, "/") if err != nil { - app.Logger.Error("Failed to initialize ASRockRack vendor:" + err.Error()) return nil, err } - app.vendors = append(app.vendors, asrr) + downloader = vendors.NewS3Downloader(app.Logger, s3Fs) case common.VendorSupermicro: - var sm vendors.Vendor - - sm, err = supermicro.New(ctx, firmwares, app.Config, app.Logger) - if err != nil { - app.Logger.Error("Failed to initialize Supermicro vendor: " + err.Error()) - return nil, err - } - - app.vendors = append(app.vendors, sm) + downloader = supermicro.NewSupermicroDownloader(app.Logger) case common.VendorMellanox: - var mlx vendors.Vendor - - mlx, err = mellanox.New(ctx, firmwares, app.Config, app.Logger) - if err != nil { - app.Logger.Error("Failed to initialize Mellanox vendor: " + err.Error()) - return nil, err - } - - app.vendors = append(app.vendors, mlx) + downloader = vendors.NewArchiveDownloader(app.Logger) case common.VendorIntel: - var i vendors.Vendor - - i, err = intel.New(ctx, firmwares, app.Config, app.Logger) - if err != nil { - app.Logger.Error("Failed to initialize Intel vendor: " + err.Error()) - return nil, err - } - - app.vendors = append(app.vendors, i) - case "equinix": - var e vendors.Vendor - - e, err = equinix.New(ctx, firmwares, app.Config, app.Logger) - if err != nil { - app.Logger.Error("Failed to initialize Equinix vendor: " + err.Error()) - return nil, err - } - - app.vendors = append(app.vendors, e) + downloader = vendors.NewArchiveDownloader(app.Logger) + case VendorEquinix: + ghClient := github.NewGitHubClient(ctx, app.Config.GithubOpenBmcToken) + downloader = github.NewGitHubDownloader(app.Logger, ghClient) default: app.Logger.Error("Vendor not supported: " + vendor) continue } + + syncer := vendors.NewSyncer(dstFs, tmpFs, downloader, inventoryClient, firmwares, app.Logger) + app.vendors = append(app.vendors, syncer) } return app, nil @@ -154,7 +117,7 @@ func (a *App) SyncFirmwares(ctx context.Context) error { for _, v := range a.vendors { err := v.Sync(ctx) if err != nil { - a.Logger.Error("Failed to sync: " + err.Error()) + a.Logger.WithError(err).Error("Failed to sync vendor") } } diff --git a/internal/inventory/mocks/serverservice.go b/internal/inventory/mocks/serverservice.go new file mode 100644 index 00000000..3d483141 --- /dev/null +++ b/internal/inventory/mocks/serverservice.go @@ -0,0 +1,54 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: serverservice.go +// +// Generated by this command: +// +// mockgen -source=serverservice.go -destination=mocks/serverservice.go ServerService +// +// Package mock_inventory is a generated GoMock package. +package mock_inventory + +import ( + context "context" + reflect "reflect" + + serverservice "go.hollow.sh/serverservice/pkg/api/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockServerService is a mock of ServerService interface. +type MockServerService struct { + ctrl *gomock.Controller + recorder *MockServerServiceMockRecorder +} + +// MockServerServiceMockRecorder is the mock recorder for MockServerService. +type MockServerServiceMockRecorder struct { + mock *MockServerService +} + +// NewMockServerService creates a new mock instance. +func NewMockServerService(ctrl *gomock.Controller) *MockServerService { + mock := &MockServerService{ctrl: ctrl} + mock.recorder = &MockServerServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServerService) EXPECT() *MockServerServiceMockRecorder { + return m.recorder +} + +// Publish mocks base method. +func (m *MockServerService) Publish(ctx context.Context, newFirmware *serverservice.ComponentFirmwareVersion) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, newFirmware) + ret0, _ := ret[0].(error) + return ret0 +} + +// Publish indicates an expected call of Publish. +func (mr *MockServerServiceMockRecorder) Publish(ctx, newFirmware any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockServerService)(nil).Publish), ctx, newFirmware) +} diff --git a/internal/inventory/serverservice.go b/internal/inventory/serverservice.go index 1a511d16..fd4ced0d 100644 --- a/internal/inventory/serverservice.go +++ b/internal/inventory/serverservice.go @@ -3,17 +3,16 @@ package inventory import ( "context" "net/url" + "path" "strings" "github.com/coreos/go-oidc" - "github.com/google/uuid" - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" "golang.org/x/oauth2/clientcredentials" + "github.com/metal-toolbox/firmware-syncer/internal/config" + serverservice "go.hollow.sh/serverservice/pkg/api/v1" ) @@ -22,19 +21,25 @@ var ( ErrServerServiceQuery = errors.New("server service query failed") ) -type ServerService struct { +//go:generate mockgen -source=serverservice.go -destination=mocks/serverservice.go ServerService + +type ServerService interface { + Publish(ctx context.Context, newFirmware *serverservice.ComponentFirmwareVersion) error +} + +type serverService struct { artifactsURL string client *serverservice.Client logger *logrus.Logger } -func New(ctx context.Context, cfg *config.ServerserviceOptions, artifactsURL string, logger *logrus.Logger) (*ServerService, error) { +func New(ctx context.Context, cfg *config.ServerserviceOptions, artifactsURL string, logger *logrus.Logger) (ServerService, error) { var client *serverservice.Client var err error if !cfg.DisableOAuth { - client, err = newClientWithOAuth(ctx, cfg, logger) + client, err = newClientWithOAuth(ctx, cfg) if err != nil { return nil, err } @@ -45,14 +50,14 @@ func New(ctx context.Context, cfg *config.ServerserviceOptions, artifactsURL str } } - return &ServerService{ + return &serverService{ artifactsURL: artifactsURL, client: client, logger: logger, }, nil } -func newClientWithOAuth(ctx context.Context, cfg *config.ServerserviceOptions, logger *logrus.Logger) (client *serverservice.Client, err error) { +func newClientWithOAuth(ctx context.Context, cfg *config.ServerserviceOptions) (client *serverservice.Client, err error) { provider, err := oidc.NewProvider(ctx, cfg.OidcIssuerEndpoint) if err != nil { return nil, err @@ -74,20 +79,21 @@ func newClientWithOAuth(ctx context.Context, cfg *config.ServerserviceOptions, l return client, nil } -// nolint:gocyclo // silence cyclo warning +func makeFirmwarePath(fw *serverservice.ComponentFirmwareVersion) string { + return path.Join(fw.Vendor, fw.Filename) +} + // Publish adds firmware data to Hollow's ServerService -func (s *ServerService) Publish(ctx context.Context, cfv *serverservice.ComponentFirmwareVersion) error { - artifactsURL, err := url.JoinPath(s.artifactsURL, vendors.DstPath(cfv)) +func (s *serverService) Publish(ctx context.Context, newFirmware *serverservice.ComponentFirmwareVersion) error { + artifactsURL, err := url.JoinPath(s.artifactsURL, makeFirmwarePath(newFirmware)) if err != nil { return err } - cfv.RepositoryURL = artifactsURL + newFirmware.RepositoryURL = artifactsURL params := serverservice.ComponentFirmwareVersionListParams{ - Vendor: cfv.Vendor, - Version: cfv.Version, - Filename: cfv.Filename, + Checksum: newFirmware.Checksum, } firmwares, _, err := s.client.ListServerComponentFirmware(ctx, ¶ms) @@ -95,72 +101,102 @@ func (s *ServerService) Publish(ctx context.Context, cfv *serverservice.Componen return errors.Wrap(ErrServerServiceQuery, "ListServerComponentFirmware: "+err.Error()) } - if len(firmwares) == 0 { - var u *uuid.UUID + firmwareCount := len(firmwares) - u, _, err = s.client.CreateServerComponentFirmware(ctx, *cfv) - if err != nil { - return errors.Wrap(ErrServerServiceQuery, "CreateServerComponentFirmware: "+err.Error()) - } + if firmwareCount == 0 { + return s.createFirmware(ctx, newFirmware) + } - s.logger.WithFields( - logrus.Fields{ - "uuid": u, - }, - ).Info("published firmware") - - return nil - } - - if len(firmwares) == 1 { - // check if the firmware already includes this model - var update bool - - for _, m := range cfv.Model { - if !slices.Contains(firmwares[0].Model, m) { - firmwares[0].Model = append(firmwares[0].Model, m) - update = true - } else { - s.logger.WithFields( - logrus.Fields{ - "uuid": &firmwares[0].UUID, - "vendor": firmwares[0].Vendor, - "model": cfv.Model, - "version": cfv.Version, - }, - ).Info("firmware already published for model") - } + if firmwareCount != 1 { + uuids := make([]string, len(firmwares)) + for i := range firmwares { + uuids[i] = firmwares[i].UUID.String() } - // Submit changed firmware to server service - if update { - _, err = s.client.UpdateServerComponentFirmware(ctx, firmwares[0].UUID, firmwares[0]) - if err != nil { - return errors.Wrap(ErrServerServiceQuery, "UpdateServerComponentFirmware: "+err.Error()) - } - - s.logger.WithFields( - logrus.Fields{ - "uuid": &firmwares[0].UUID, - "model": &firmwares[0].Model, - }, - ).Info("firmware updated with new models") - } + uuidLog := strings.Join(uuids, ",") + + s.logger.WithField("uuids", uuidLog). + WithField("checksum", newFirmware.Checksum). + Error("Multiple firmware IDs found with checksum") + + return errors.Wrap(ErrServerServiceDuplicateFirmware, uuidLog) + } + + newFirmware.UUID = firmwares[0].UUID + + if isDifferent(newFirmware, &firmwares[0]) { + return s.updateFirmware(ctx, newFirmware) + } + + s.logger.WithField("firmware", newFirmware.Filename). + WithField("vendor", newFirmware.Vendor). + Debug("Firmware already exists and is up to date") + + return nil +} + +func isDifferent(firmware1, firmware2 *serverservice.ComponentFirmwareVersion) bool { + if firmware1.Vendor != firmware2.Vendor { + return true + } - return nil + if firmware1.Filename != firmware2.Filename { + return true } - // Assumption at this point is that there are duplicated firmwares returned by the ListServerComponentFirmware query. - uuids := make([]string, len(firmwares)) - for i := range firmwares { - uuids[i] = firmwares[i].UUID.String() + if firmware1.Version != firmware2.Version { + return true + } + + if firmware1.Component != firmware2.Component { + return true + } + + if firmware1.Checksum != firmware2.Checksum { + return true + } + + if firmware1.UpstreamURL != firmware2.UpstreamURL { + return true + } + + if firmware1.RepositoryURL != firmware2.RepositoryURL { + return true + } + + if strings.Join(firmware1.Model, ",") != strings.Join(firmware2.Model, ",") { + return true + } + + return false +} + +func (s *serverService) createFirmware(ctx context.Context, firmware *serverservice.ComponentFirmwareVersion) error { + id, response, err := s.client.CreateServerComponentFirmware(ctx, *firmware) + + if err != nil { + return errors.Wrap(ErrServerServiceQuery, "CreateServerComponentFirmware: "+err.Error()) + } + + s.logger.WithField("response", response). + WithField("firmware", firmware.Filename). + WithField("vendor", firmware.Vendor). + WithField("uuid", id). + Info("Created firmware") + + return nil +} + +func (s *serverService) updateFirmware(ctx context.Context, firmware *serverservice.ComponentFirmwareVersion) error { + response, err := s.client.UpdateServerComponentFirmware(ctx, firmware.UUID, *firmware) + if err != nil { + return errors.Wrap(ErrServerServiceQuery, "UpdateServerComponentFirmware: "+err.Error()) } - s.logger.WithFields( - logrus.Fields{ - "uuids": strings.Join(uuids, ","), - }, - ).Info("duplicate firmware IDs") + s.logger.WithField("firmware", firmware.Filename). + WithField("vendor", firmware.Vendor). + WithField("response", response). + Info("Updated firmware") - return errors.Wrap(ErrServerServiceDuplicateFirmware, strings.Join(uuids, ",")) + return nil } diff --git a/internal/inventory/serverservice_test.go b/internal/inventory/serverservice_test.go index f8b11088..75f211a2 100644 --- a/internal/inventory/serverservice_test.go +++ b/internal/inventory/serverservice_test.go @@ -47,6 +47,7 @@ func TestServerServicePublish(t *testing.T) { ) mock := httptest.NewServer(handler) + defer mock.Close() cfg := config.ServerserviceOptions{ Endpoint: mock.URL, diff --git a/internal/logging/logger.go b/internal/logging/logger.go new file mode 100644 index 00000000..81a6b7a1 --- /dev/null +++ b/internal/logging/logger.go @@ -0,0 +1,33 @@ +package logging + +import ( + runtime "github.com/banzaicloud/logrus-runtime-formatter" + "github.com/sirupsen/logrus" + + "github.com/metal-toolbox/firmware-syncer/pkg/types" +) + +// NewLogger creates a new logrus.Logger with the given log level. +func NewLogger(logLevel string) *logrus.Logger { + logger := logrus.New() + + switch types.LogLevel(logLevel) { + case types.LogLevelDebug: + logger.Level = logrus.DebugLevel + case types.LogLevelTrace: + logger.Level = logrus.TraceLevel + default: + logger.Level = logrus.InfoLevel + } + + runtimeFormatter := &runtime.Formatter{ + ChildFormatter: &logrus.JSONFormatter{}, + File: true, + Line: true, + BaseNameOnly: true, + } + + logger.SetFormatter(runtimeFormatter) + + return logger +} diff --git a/internal/vendors/asrockrack/asrockrack.go b/internal/vendors/asrockrack/asrockrack.go deleted file mode 100644 index 33ecc144..00000000 --- a/internal/vendors/asrockrack/asrockrack.go +++ /dev/null @@ -1,126 +0,0 @@ -package asrockrack - -import ( - "context" - - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" - - "github.com/pkg/errors" - rcloneFs "github.com/rclone/rclone/fs" - rcloneOperations "github.com/rclone/rclone/fs/operations" - "github.com/sirupsen/logrus" - serverservice "go.hollow.sh/serverservice/pkg/api/v1" -) - -// ASRockRack implements the Vendor interface methods to retrieve ASRockRack firmware files -type ASRockRack struct { - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService - srcCfg *config.S3Bucket - dstCfg *config.S3Bucket - srcFs rcloneFs.Fs - dstFs rcloneFs.Fs - tmpFs rcloneFs.Fs -} - -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - // init rclone filesystems for tmp, dst and src files - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - srcFs, err := vendors.InitS3Fs(ctx, cfg.AsRockRackRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } - - return &ASRockRack{ - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - srcCfg: cfg.AsRockRackRepository, - dstCfg: cfg.FirmwareRepository, - srcFs: srcFs, - dstFs: dstFs, - tmpFs: tmpFs, - }, nil -} - -func (a *ASRockRack) Stats() *vendors.Metrics { - return a.metrics -} - -func (a *ASRockRack) Sync(ctx context.Context) error { - for _, fw := range a.firmwares { - dstPath := vendors.DstPath(fw) - - a.logger.WithFields( - logrus.Fields{ - "src": fw.UpstreamURL, - "dst": dstPath, - }, - ).Info("sync ASRockRack") - - // In case the file already exists in dst, don't verify/copy it - if exists, _ := rcloneFs.FileExists(ctx, a.dstFs, dstPath); exists { - a.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - // verify file checksum - err := vendors.VerifyFile(ctx, a.tmpFs, a.srcFs, fw) - if err != nil { - return err - } - - // copy file to dst - err = a.copyFile(ctx, fw) - if err != nil { - return err - } - - err = a.inventory.Publish(ctx, fw) - if err != nil { - return err - } - } - - return nil -} - -func (a *ASRockRack) copyFile(ctx context.Context, fw *serverservice.ComponentFirmwareVersion) error { - err := rcloneOperations.CopyFile(ctx, a.dstFs, a.srcFs, vendors.DstPath(fw), vendors.SrcPath(fw)) - if err != nil { - if errors.Is(err, rcloneFs.ErrorObjectNotFound) { - return errors.Wrap(vendors.ErrCopy, err.Error()+" :"+fw.Filename) - } - - return errors.Wrap(vendors.ErrCopy, err.Error()) - } - - return nil -} diff --git a/internal/vendors/dell/dell.go b/internal/vendors/dell/dell.go deleted file mode 100644 index e1a91506..00000000 --- a/internal/vendors/dell/dell.go +++ /dev/null @@ -1,138 +0,0 @@ -package dell - -import ( - "context" - - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" - - "github.com/pkg/errors" - rcloneFs "github.com/rclone/rclone/fs" - rcloneOperations "github.com/rclone/rclone/fs/operations" - "github.com/sirupsen/logrus" - - serverservice "go.hollow.sh/serverservice/pkg/api/v1" -) - -// DUP implements the Vendor interface methods to retrieve dell DUP firmware files -type DUP struct { - dstCfg *config.S3Bucket - dstFs rcloneFs.Fs - tmpFs rcloneFs.Fs - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService -} - -// NewDUP returns a new DUP firmware syncer object -func NewDUP(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - // init rclone filesystems for tmp and dst files - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } - - return &DUP{ - dstCfg: cfg.FirmwareRepository, - dstFs: dstFs, - tmpFs: tmpFs, - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - }, nil -} - -// Stats implements the Syncer interface to return metrics collected on Object, byte transfer stats -func (d *DUP) Stats() *vendors.Metrics { - return d.metrics -} - -func (d *DUP) Sync(ctx context.Context) error { - for _, fw := range d.firmwares { - dstPath := vendors.DstPath(fw) - - d.logger.WithFields( - logrus.Fields{ - "src": fw.UpstreamURL, - "dst": dstPath, - }, - ).Info("sync DUP") - - // In case the file already exists in dst, don't verify/copy it - if exists, _ := rcloneFs.FileExists(ctx, d.dstFs, dstPath); exists { - d.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - // init src rclone filesystem - srcFs, err := d.initSrcFs(ctx, fw) - if err != nil { - return err - } - - // Verify file checksum - err = vendors.VerifyFile(ctx, d.tmpFs, srcFs, fw) - if err != nil { - return err - } - - // Copy file to dst - err = d.copyFile(ctx, fw) - if err != nil { - return err - } - - err = d.inventory.Publish(ctx, fw) - if err != nil { - return err - } - } - - return nil -} - -func (d *DUP) initSrcFs(ctx context.Context, fw *serverservice.ComponentFirmwareVersion) (srcFs rcloneFs.Fs, err error) { - // init source to download files - srcFs, err = vendors.InitHTTPFs(ctx, fw.UpstreamURL) - if err != nil { - return nil, err - } - - return srcFs, err -} - -func (d *DUP) copyFile(ctx context.Context, fw *serverservice.ComponentFirmwareVersion) error { - var err error - - _, err = rcloneOperations.CopyURL(ctx, d.dstFs, vendors.DstPath(fw), fw.UpstreamURL, false, false, false) - if err != nil { - if errors.Is(err, rcloneFs.ErrorObjectNotFound) { - return errors.Wrap(vendors.ErrCopy, err.Error()+" :"+fw.UpstreamURL) - } - - return errors.Wrap(vendors.ErrCopy, err.Error()) - } - - return nil -} diff --git a/internal/vendors/downloader.go b/internal/vendors/downloader.go index 8db44dd5..410460d0 100644 --- a/internal/vendors/downloader.go +++ b/internal/vendors/downloader.go @@ -11,11 +11,11 @@ import ( "path/filepath" "strings" - "github.com/metal-toolbox/firmware-syncer/internal/config" "github.com/pkg/errors" "github.com/sirupsen/logrus" - rcloneHttp "github.com/rclone/rclone/backend/http" + "github.com/metal-toolbox/firmware-syncer/internal/config" + rcloneLocal "github.com/rclone/rclone/backend/local" rcloneS3 "github.com/rclone/rclone/backend/s3" rcloneFs "github.com/rclone/rclone/fs" @@ -48,6 +48,15 @@ var ( ErrCreatingTmpDir = errors.New("error creating tmp dir") ) +//go:generate mockgen -source=downloader.go -destination=mocks/downloader.go Downloader + +// Downloader is something that can download a file for a given firmware. +type Downloader interface { + // Download takes in the directory to download the file to, and the firmware to be downloaded. + // It should also return the full path to the downloaded file. + Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) +} + // DownloaderStats includes fields for stats on file/object transfer for Downloader type DownloaderStats struct { BytesTransferred int64 @@ -79,71 +88,7 @@ func DstPath(fw *serverservice.ComponentFirmwareVersion) string { return path.Join(fw.Vendor, fw.Filename) } -func VerifyFile(ctx context.Context, tmpFs, srcFs rcloneFs.Fs, fw *serverservice.ComponentFirmwareVersion) error { - // create local tmp directory - tmpDir, err := os.MkdirTemp(tmpFs.Root(), "verify-") - if err != nil { - return errors.Wrap(ErrCreatingTmpDir, err.Error()) - } - - defer os.RemoveAll(tmpDir) - - dstPath := path.Join(path.Base(tmpDir), fw.Filename) - - switch { - case strings.HasPrefix(fw.UpstreamURL, "s3://"): - err = rcloneOperations.CopyFile(ctx, tmpFs, srcFs, dstPath, SrcPath(fw)) - case strings.HasPrefix(fw.UpstreamURL, "http://"), strings.HasPrefix(fw.UpstreamURL, "https://"): - _, err = rcloneOperations.CopyURL(ctx, tmpFs, dstPath, fw.UpstreamURL, false, false, false) - } - - if err != nil { - if errors.Is(err, rcloneFs.ErrorObjectNotFound) { - return errors.Wrap(ErrCopy, err.Error()+" :"+fw.Filename) - } - - return errors.Wrap(ErrCopy, err.Error()) - } - - tmpFilename := path.Join(tmpFs.Root(), dstPath) - - if !ValidateChecksum(tmpFilename, fw.Checksum) { - return errors.Wrap(ErrChecksumValidate, fmt.Sprintf("tmpFilename: %s, expected checksum: %s", tmpFilename, fw.Checksum)) - } - - return nil -} - -// initHttpFs initializes and returns a rcloneFs.Fs interface that can be used for Copy, Sync operations -// the Fs is initialized based the urlHost, urlPath parameters -// -// httpURL: the http endpoint which is expected to be the root/top level directory from where files are to be copied from/to -// -// this can be a http index or a URL endpoint from which files are to be downloaded. -func InitHTTPFs(ctx context.Context, httpURL string) (rcloneFs.Fs, error) { - // parse the URL into host and path parts, as expected by the rclone fs lib - hostPart, pathPart, err := SplitURLPath(httpURL) - if err != nil { - return nil, err - } - - // https://github.com/rclone/rclone/blob/master/backend/http/http.go#L36 - opts := rcloneConfigmap.Simple{ - "type": "http", - "no_head": "true", - "url": hostPart, - } - - fs, err := rcloneHttp.NewFs(ctx, httpURL, pathPart, opts) - - if err != nil && !errors.Is(err, rcloneFs.ErrorIsFile) { - return nil, errors.Wrap(ErrInitHTTPDownloader, err.Error()) - } - - return fs, nil -} - -// initLocalFs initializes and returns a rcloneFs.Fs interface on the local filesystem +// InitLocalFs initializes and returns a rcloneFs.Fs interface on the local filesystem func InitLocalFs(ctx context.Context, cfg *LocalFsConfig) (rcloneFs.Fs, error) { if cfg == nil { return nil, errors.Wrap(ErrFileStoreConfig, "got nil local fs config") @@ -172,7 +117,7 @@ func InitLocalFs(ctx context.Context, cfg *LocalFsConfig) (rcloneFs.Fs, error) { return fs, nil } -// initS3Fs initializes and returns a rcloneFs.Fs interface on an s3 store +// InitS3Fs initializes and returns a rcloneFs.Fs interface on an s3 store // // root: the directory mounted as the root/top level directory of the returned fs func InitS3Fs(ctx context.Context, cfg *config.S3Bucket, root string) (rcloneFs.Fs, error) { @@ -348,3 +293,77 @@ func ExtractFromZipArchive(archivePath, firmwareFilename, firmwareChecksum strin return out, nil } + +type ArchiveDownloader struct { + logger *logrus.Logger +} + +// NewArchiveDownloader creates a new ArchiveDownloader. +func NewArchiveDownloader(logger *logrus.Logger) Downloader { + return &ArchiveDownloader{logger: logger} +} + +// Download will download the file for the given firmware into the given downloadDir, +// and return the full path to the downloaded file. +func (m *ArchiveDownloader) Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) { + archivePath, err := DownloadFirmwareArchive(ctx, downloadDir, firmware.UpstreamURL, "") + if err != nil { + return "", err + } + + m.logger.WithField("archivePath", archivePath).Debug("Archive downloaded.") + m.logger.Debug("Extracting firmware from archive") + + fwFile, err := ExtractFromZipArchive(archivePath, firmware.Filename, "") + if err != nil { + return "", err + } + + return fwFile.Name(), nil +} + +type RcloneDownloader struct { + logger *logrus.Logger +} + +// NewRcloneDownloader creates a new RcloneDownloader. +func NewRcloneDownloader(logger *logrus.Logger) Downloader { + return &RcloneDownloader{logger: logger} +} + +// Download will download the file for the given firmware into the given downloadDir, +// and return the full path to the downloaded file. +func (r *RcloneDownloader) Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) { + return DownloadFirmwareArchive(ctx, downloadDir, firmware.UpstreamURL, "") +} + +type S3Downloader struct { + logger *logrus.Logger + s3Fs rcloneFs.Fs +} + +// NewS3Downloader creats a new S3Downloader. +func NewS3Downloader(logger *logrus.Logger, s3Fs rcloneFs.Fs) Downloader { + return &S3Downloader{logger: logger, s3Fs: s3Fs} +} + +// Download will download the file for the given firmware into the given downloadDir, +// and return the full path to the downloaded file. +func (s *S3Downloader) Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) { + tmpFS, err := InitLocalFs(ctx, &LocalFsConfig{Root: downloadDir}) + if err != nil { + return "", err + } + + err = rcloneOperations.CopyFile(ctx, tmpFS, s.s3Fs, firmware.Filename, SrcPath(firmware)) + if err != nil { + if errors.Is(err, rcloneFs.ErrorObjectNotFound) { + msg := fmt.Sprintf("%s: %s", err, firmware.Filename) + return "", errors.Wrap(ErrCopy, msg) + } + + return "", err + } + + return path.Join(downloadDir, firmware.Filename), nil +} diff --git a/internal/vendors/equinix/equinix.go b/internal/vendors/equinix/equinix.go deleted file mode 100644 index fd8cf761..00000000 --- a/internal/vendors/equinix/equinix.go +++ /dev/null @@ -1,186 +0,0 @@ -package equinix - -import ( - "context" - "fmt" - "net/http" - "net/url" - "strings" - "time" - - "github.com/google/go-github/v53/github" - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" - "golang.org/x/oauth2" - - "github.com/pkg/errors" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/operations" - "github.com/sirupsen/logrus" - serverservice "go.hollow.sh/serverservice/pkg/api/v1" -) - -const GithubDownloadTimeout = 300 - -// Equinix implements the Vendor interface methods to retrieve Equinix OpenBMC firmware files -type Equinix struct { - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService - ghClient *github.Client - dstCfg *config.S3Bucket - dstFs fs.Fs - tmpFs fs.Fs -} - -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: cfg.GithubOpenBmcToken}, - ) - tc := oauth2.NewClient(ctx, ts) - - ghClient := github.NewClient(tc) - - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - // init rclone filesystems for tmp and dst files - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } - - return &Equinix{ - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - ghClient: ghClient, - dstCfg: cfg.FirmwareRepository, - dstFs: dstFs, - tmpFs: tmpFs, - }, nil -} - -func (e *Equinix) Stats() *vendors.Metrics { - return e.metrics -} - -func (e *Equinix) Sync(ctx context.Context) error { - for _, fw := range e.firmwares { - // In case the file already exists in dst, don't copy it - if exists, _ := fs.FileExists(ctx, e.dstFs, vendors.DstPath(fw)); exists { - e.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - err := e.getFileFromGithub(ctx, fw) - if err != nil { - return err - } - - // Verify file checksum - tmpFilename := e.tmpFs.Root() + "/" + fw.Filename - if !vendors.ValidateChecksum(tmpFilename, fw.Checksum) { - return errors.Wrap(vendors.ErrChecksumValidate, fmt.Sprintf("tmpFilename: %s, expected checksum: %s", tmpFilename, fw.Checksum)) - } - - e.logger.WithFields( - logrus.Fields{ - "src": fw.UpstreamURL, - "dst": vendors.DstPath(fw), - }, - ).Info("sync Equinix") - - // Copy from tmpfs to dstfs - err = operations.CopyFile(ctx, e.dstFs, e.tmpFs, vendors.DstPath(fw), fw.Filename) - if err != nil { - return err - } - - err = e.inventory.Publish(ctx, fw) - if err != nil { - return err - } - } - - return nil -} - -func (e *Equinix) getFileFromGithub(ctx context.Context, fw *serverservice.ComponentFirmwareVersion) error { - owner, repo, tag, filename, err := parseGithubReleaseURL(fw.UpstreamURL) - if err != nil { - return err - } - - release, _, err := e.ghClient.Repositories.GetReleaseByTag(ctx, owner, repo, tag) - if err != nil { - return err - } - - asset, err := getAssetByName(filename, release.Assets) - if err != nil { - return err - } - - // Give enough time for the client to download the binary file. - redirectClient := &http.Client{ - Timeout: time.Second * GithubDownloadTimeout, - } - - rc, _, err := e.ghClient.Repositories.DownloadReleaseAsset(ctx, owner, repo, *asset.ID, redirectClient) - if err != nil { - return err - } - defer rc.Close() - - // Copy downloaded file to tmpFs for checksum verification and later upload to dst - _, err = operations.Rcat(ctx, e.tmpFs, fw.Filename, rc, time.Now(), nil) - if err != nil { - return err - } - - return nil -} - -func parseGithubReleaseURL(ghURL string) (owner, repo, release, filename string, err error) { - // https://github.com///releases/download// - u, err := url.Parse(ghURL) - if err != nil { - return "", "", "", "", err - } - - components := strings.Split(u.Path, "/") - if len(components) != 7 { - return "", "", "", "", errors.New(fmt.Sprintf("parsing failed for URL path: %s", u.Path)) - } - - return components[1], components[2], components[5], components[6], nil -} - -func getAssetByName(assetName string, assets []*github.ReleaseAsset) (asset *github.ReleaseAsset, err error) { - for _, a := range assets { - if assetName == *a.Name { - return a, nil - } - } - - return nil, errors.New("asset doesn't exist with given name") -} diff --git a/internal/vendors/fixtures/foobar.zip b/internal/vendors/fixtures/foobar.zip new file mode 100644 index 0000000000000000000000000000000000000000..402206f0bfced9a59fda2d5ce249fc03847cfbd1 GIT binary patch literal 177 zcmWIWW@h1H009}V_E7ifI-A&mY!K#RkYPy6&reD$(o4$B3k~69VE*nsAsK{AE4UdL zS-vweFtC7hLR4`Dcr!A|G2=2!0;G}wsD)ukBZ!4;G%LhtG-Ct2S=m4;8G$eWNPB}g F3;=Q!Ab9`) literal 0 HcmV?d00001 diff --git a/internal/vendors/github/github.go b/internal/vendors/github/github.go new file mode 100644 index 00000000..f099b565 --- /dev/null +++ b/internal/vendors/github/github.go @@ -0,0 +1,117 @@ +package github + +import ( + "context" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/google/go-github/v53/github" + "github.com/pkg/errors" + "github.com/rclone/rclone/fs/operations" + "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + + "github.com/metal-toolbox/firmware-syncer/internal/vendors" + + serverservice "go.hollow.sh/serverservice/pkg/api/v1" +) + +const DownloadTimeout = 300 + +// NewGitHubClient creates a new github.Client. +func NewGitHubClient(ctx context.Context, githubOpenBmcToken string) *github.Client { + tokenSource := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: githubOpenBmcToken}, + ) + tokenClient := oauth2.NewClient(ctx, tokenSource) + + return github.NewClient(tokenClient) +} + +type Downloader struct { + logger *logrus.Logger + client *github.Client +} + +// NewGitHubDownloader creates a new vendors.Downloader that can download content from GitHub. +func NewGitHubDownloader(logger *logrus.Logger, client *github.Client) vendors.Downloader { + return &Downloader{ + logger: logger, + client: client, + } +} + +// Download will download the file for the given firmware from GitHub into the downloadDir, +// and returns the full path to the downloaded file. +func (d *Downloader) Download( + ctx context.Context, + downloadDir string, + firmware *serverservice.ComponentFirmwareVersion, +) (string, error) { + tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: downloadDir}) + if err != nil { + return "", err + } + + owner, repo, tag, filename, err := parseGithubReleaseURL(firmware.UpstreamURL) + if err != nil { + return "", err + } + + release, _, err := d.client.Repositories.GetReleaseByTag(ctx, owner, repo, tag) + if err != nil { + return "", err + } + + asset, err := getAssetByName(filename, release.Assets) + if err != nil { + return "", err + } + + // Give enough time for the client to download the binary file. + redirectClient := &http.Client{ + Timeout: time.Second * DownloadTimeout, + } + + rc, _, err := d.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, *asset.ID, redirectClient) + if err != nil { + return "", err + } + defer rc.Close() + + _, err = operations.Rcat(ctx, tmpFs, firmware.Filename, rc, time.Now(), nil) + if err != nil { + return "", err + } + + return path.Join(downloadDir, firmware.Filename), nil +} + +func parseGithubReleaseURL(ghURL string) (owner, repo, release, filename string, err error) { + // https://github.com///releases/download// + u, err := url.Parse(ghURL) + if err != nil { + return "", "", "", "", err + } + + components := strings.Split(u.Path, "/") + if len(components) != 7 { + return "", "", "", "", errors.New(fmt.Sprintf("parsing failed for URL path: %s", u.Path)) + } + + return components[1], components[2], components[5], components[6], nil +} + +func getAssetByName(assetName string, assets []*github.ReleaseAsset) (asset *github.ReleaseAsset, err error) { + for _, a := range assets { + if assetName == *a.Name { + return a, nil + } + } + + return nil, errors.New("asset doesn't exist with given name") +} diff --git a/internal/vendors/equinix/equinix_test.go b/internal/vendors/github/github_test.go similarity index 98% rename from internal/vendors/equinix/equinix_test.go rename to internal/vendors/github/github_test.go index 85511ee7..c005146e 100644 --- a/internal/vendors/equinix/equinix_test.go +++ b/internal/vendors/github/github_test.go @@ -1,4 +1,4 @@ -package equinix +package github import ( "testing" diff --git a/internal/vendors/intel/intel.go b/internal/vendors/intel/intel.go deleted file mode 100644 index bce3e9f4..00000000 --- a/internal/vendors/intel/intel.go +++ /dev/null @@ -1,136 +0,0 @@ -package intel - -import ( - "context" - "os" - "strings" - - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/operations" - "github.com/sirupsen/logrus" - - serverservice "go.hollow.sh/serverservice/pkg/api/v1" -) - -type Intel struct { - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService - dstCfg *config.S3Bucket - dstFs fs.Fs - tmpFs fs.Fs -} - -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } - - return &Intel{ - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - dstCfg: cfg.FirmwareRepository, - dstFs: dstFs, - tmpFs: tmpFs, - }, nil -} - -func (i *Intel) Stats() *vendors.Metrics { - return i.metrics -} - -// Sync copies firmware files from Intel and publishes to ServerService -// Initially only supports network card firmware for a given NIC family (e.g. E810, X710) -// Each NIC family may have multiple firmware binaries that applies to specific models within the family. -// The NVM update utility is also provided in the tarball downloaded and extracted from the zip archive. -func (i *Intel) Sync(ctx context.Context) error { - for _, fw := range i.firmwares { - // In case the file already exists in dst, don't copy it - if exists, _ := fs.FileExists(ctx, i.dstFs, vendors.DstPath(fw)); exists { - i.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - // initialize a tmpDir so we can download and unpack the zip archive - tmpDir, err := os.MkdirTemp(i.tmpFs.Root(), "firmware-archive") - if err != nil { - return err - } - - i.logger.Debug("Downloading archive") - - archivePath, err := vendors.DownloadFirmwareArchive(ctx, tmpDir, fw.UpstreamURL, "") - if err != nil { - return err - } - - i.logger.WithFields( - logrus.Fields{ - "archivePath": archivePath, - }, - ).Debug("Archive downloaded.") - - i.logger.Debug("Extracting firmware from archive") - - fwFile, err := vendors.ExtractFromZipArchive(archivePath, fw.Filename, fw.Checksum) - if err != nil { - return err - } - - i.logger.WithFields( - logrus.Fields{ - "fwFile": fwFile.Name(), - }, - ).Debug("Firmware extracted.") - - i.logger.WithFields( - logrus.Fields{ - "src": fwFile.Name(), - "dst": vendors.DstPath(fw), - }, - ).Info("Sync Intel") - - // Remove root of tmpdir from filename since CopyFile doesn't use it - tmpFwPath := strings.Replace(fwFile.Name(), i.tmpFs.Root(), "", 1) - - err = operations.CopyFile(ctx, i.dstFs, i.tmpFs, vendors.DstPath(fw), tmpFwPath) - if err != nil { - return err - } - - // Clean up tmpDir after copying the extracted firmware to dst. - os.RemoveAll(tmpDir) - - err = i.inventory.Publish(ctx, fw) - if err != nil { - return err - } - } - - return nil -} diff --git a/internal/vendors/mellanox/mellanox.go b/internal/vendors/mellanox/mellanox.go deleted file mode 100644 index f18c4f34..00000000 --- a/internal/vendors/mellanox/mellanox.go +++ /dev/null @@ -1,131 +0,0 @@ -package mellanox - -import ( - "context" - "os" - "strings" - - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" - rcloneFs "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/operations" - "github.com/sirupsen/logrus" - serverservice "go.hollow.sh/serverservice/pkg/api/v1" -) - -type Mellanox struct { - dstCfg *config.S3Bucket - dstFs rcloneFs.Fs - tmpFs rcloneFs.Fs - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService -} - -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } - - return &Mellanox{ - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - dstCfg: cfg.FirmwareRepository, - dstFs: dstFs, - tmpFs: tmpFs, - }, nil -} - -func (m *Mellanox) Stats() *vendors.Metrics { - return m.metrics -} - -func (m *Mellanox) Sync(ctx context.Context) error { - for _, fw := range m.firmwares { - // In case the file already exists in dst, don't copy it - if exists, _ := rcloneFs.FileExists(ctx, m.dstFs, vendors.DstPath(fw)); exists { - m.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - // initialize a tmpDir so we can download and unpack the zip archive - tmpDir, err := os.MkdirTemp(m.tmpFs.Root(), "firmware-archive") - if err != nil { - return err - } - - m.logger.Debug("Downloading archive") - - archivePath, err := vendors.DownloadFirmwareArchive(ctx, tmpDir, fw.UpstreamURL, "") - if err != nil { - return err - } - - m.logger.WithFields( - logrus.Fields{ - "archivePath": archivePath, - }, - ).Debug("Archive downloaded.") - - m.logger.Debug("Extracting firmware from archive") - - fwFile, err := vendors.ExtractFromZipArchive(archivePath, fw.Filename, fw.Checksum) - if err != nil { - return err - } - - m.logger.WithFields( - logrus.Fields{ - "fwFile": fwFile.Name(), - }, - ).Debug("Firmware extracted.") - - m.logger.WithFields( - logrus.Fields{ - "src": fwFile.Name(), - "dst": vendors.DstPath(fw), - }, - ).Info("Sync Mellanox") - - // Remove root of tmpdir from filename since CopyFile doesn't use it - tmpFwPath := strings.Replace(fwFile.Name(), m.tmpFs.Root(), "", 1) - - err = operations.CopyFile(ctx, m.dstFs, m.tmpFs, vendors.DstPath(fw), tmpFwPath) - if err != nil { - return err - } - - // Clean up tmpDir after copying the extracted firmware to dst. - os.RemoveAll(tmpDir) - - err = m.inventory.Publish(ctx, fw) - if err != nil { - return err - } - } - - return nil -} diff --git a/internal/vendors/mocks/downloader.go b/internal/vendors/mocks/downloader.go new file mode 100644 index 00000000..641749b5 --- /dev/null +++ b/internal/vendors/mocks/downloader.go @@ -0,0 +1,55 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: downloader.go +// +// Generated by this command: +// +// mockgen -source=downloader.go -destination=mocks/downloader.go Downloader +// +// Package mock_vendors is a generated GoMock package. +package mock_vendors + +import ( + context "context" + reflect "reflect" + + serverservice "go.hollow.sh/serverservice/pkg/api/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockDownloader is a mock of Downloader interface. +type MockDownloader struct { + ctrl *gomock.Controller + recorder *MockDownloaderMockRecorder +} + +// MockDownloaderMockRecorder is the mock recorder for MockDownloader. +type MockDownloaderMockRecorder struct { + mock *MockDownloader +} + +// NewMockDownloader creates a new mock instance. +func NewMockDownloader(ctrl *gomock.Controller) *MockDownloader { + mock := &MockDownloader{ctrl: ctrl} + mock.recorder = &MockDownloaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDownloader) EXPECT() *MockDownloaderMockRecorder { + return m.recorder +} + +// Download mocks base method. +func (m *MockDownloader) Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Download", ctx, downloadDir, firmware) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Download indicates an expected call of Download. +func (mr *MockDownloaderMockRecorder) Download(ctx, downloadDir, firmware any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Download", reflect.TypeOf((*MockDownloader)(nil).Download), ctx, downloadDir, firmware) +} diff --git a/internal/vendors/mocks/rclone.go b/internal/vendors/mocks/rclone.go new file mode 100644 index 00000000..71ae90d7 --- /dev/null +++ b/internal/vendors/mocks/rclone.go @@ -0,0 +1,501 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: syncer_test.go +// +// Generated by this command: +// +// mockgen -source=syncer_test.go -destination=mocks/rclone.go RCloneInfo +// +// Package mock_vendors is a generated GoMock package. +package mock_vendors + +import ( + context "context" + io "io" + reflect "reflect" + time "time" + + fs "github.com/rclone/rclone/fs" + hash "github.com/rclone/rclone/fs/hash" + gomock "go.uber.org/mock/gomock" +) + +// MockRCloneFS is a mock of RCloneFS interface. +type MockRCloneFS struct { + ctrl *gomock.Controller + recorder *MockRCloneFSMockRecorder +} + +// MockRCloneFSMockRecorder is the mock recorder for MockRCloneFS. +type MockRCloneFSMockRecorder struct { + mock *MockRCloneFS +} + +// NewMockRCloneFS creates a new mock instance. +func NewMockRCloneFS(ctrl *gomock.Controller) *MockRCloneFS { + mock := &MockRCloneFS{ctrl: ctrl} + mock.recorder = &MockRCloneFSMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRCloneFS) EXPECT() *MockRCloneFSMockRecorder { + return m.recorder +} + +// Features mocks base method. +func (m *MockRCloneFS) Features() *fs.Features { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Features") + ret0, _ := ret[0].(*fs.Features) + return ret0 +} + +// Features indicates an expected call of Features. +func (mr *MockRCloneFSMockRecorder) Features() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Features", reflect.TypeOf((*MockRCloneFS)(nil).Features)) +} + +// Hashes mocks base method. +func (m *MockRCloneFS) Hashes() hash.Set { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Hashes") + ret0, _ := ret[0].(hash.Set) + return ret0 +} + +// Hashes indicates an expected call of Hashes. +func (mr *MockRCloneFSMockRecorder) Hashes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Hashes", reflect.TypeOf((*MockRCloneFS)(nil).Hashes)) +} + +// List mocks base method. +func (m *MockRCloneFS) List(ctx context.Context, dir string) (fs.DirEntries, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, dir) + ret0, _ := ret[0].(fs.DirEntries) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockRCloneFSMockRecorder) List(ctx, dir any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRCloneFS)(nil).List), ctx, dir) +} + +// Mkdir mocks base method. +func (m *MockRCloneFS) Mkdir(ctx context.Context, dir string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Mkdir", ctx, dir) + ret0, _ := ret[0].(error) + return ret0 +} + +// Mkdir indicates an expected call of Mkdir. +func (mr *MockRCloneFSMockRecorder) Mkdir(ctx, dir any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mkdir", reflect.TypeOf((*MockRCloneFS)(nil).Mkdir), ctx, dir) +} + +// Name mocks base method. +func (m *MockRCloneFS) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockRCloneFSMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockRCloneFS)(nil).Name)) +} + +// NewObject mocks base method. +func (m *MockRCloneFS) NewObject(ctx context.Context, remote string) (fs.Object, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewObject", ctx, remote) + ret0, _ := ret[0].(fs.Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewObject indicates an expected call of NewObject. +func (mr *MockRCloneFSMockRecorder) NewObject(ctx, remote any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewObject", reflect.TypeOf((*MockRCloneFS)(nil).NewObject), ctx, remote) +} + +// Precision mocks base method. +func (m *MockRCloneFS) Precision() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Precision") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// Precision indicates an expected call of Precision. +func (mr *MockRCloneFSMockRecorder) Precision() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Precision", reflect.TypeOf((*MockRCloneFS)(nil).Precision)) +} + +// Put mocks base method. +func (m *MockRCloneFS) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in, src} + for _, a := range options { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Put", varargs...) + ret0, _ := ret[0].(fs.Object) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Put indicates an expected call of Put. +func (mr *MockRCloneFSMockRecorder) Put(ctx, in, src any, options ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in, src}, options...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockRCloneFS)(nil).Put), varargs...) +} + +// Rmdir mocks base method. +func (m *MockRCloneFS) Rmdir(ctx context.Context, dir string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Rmdir", ctx, dir) + ret0, _ := ret[0].(error) + return ret0 +} + +// Rmdir indicates an expected call of Rmdir. +func (mr *MockRCloneFSMockRecorder) Rmdir(ctx, dir any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rmdir", reflect.TypeOf((*MockRCloneFS)(nil).Rmdir), ctx, dir) +} + +// Root mocks base method. +func (m *MockRCloneFS) Root() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Root") + ret0, _ := ret[0].(string) + return ret0 +} + +// Root indicates an expected call of Root. +func (mr *MockRCloneFSMockRecorder) Root() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockRCloneFS)(nil).Root)) +} + +// String mocks base method. +func (m *MockRCloneFS) String() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String") + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockRCloneFSMockRecorder) String() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockRCloneFS)(nil).String)) +} + +// MockRCloneObject is a mock of RCloneObject interface. +type MockRCloneObject struct { + ctrl *gomock.Controller + recorder *MockRCloneObjectMockRecorder +} + +// MockRCloneObjectMockRecorder is the mock recorder for MockRCloneObject. +type MockRCloneObjectMockRecorder struct { + mock *MockRCloneObject +} + +// NewMockRCloneObject creates a new mock instance. +func NewMockRCloneObject(ctrl *gomock.Controller) *MockRCloneObject { + mock := &MockRCloneObject{ctrl: ctrl} + mock.recorder = &MockRCloneObjectMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRCloneObject) EXPECT() *MockRCloneObjectMockRecorder { + return m.recorder +} + +// Fs mocks base method. +func (m *MockRCloneObject) Fs() fs.Info { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Fs") + ret0, _ := ret[0].(fs.Info) + return ret0 +} + +// Fs indicates an expected call of Fs. +func (mr *MockRCloneObjectMockRecorder) Fs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Fs", reflect.TypeOf((*MockRCloneObject)(nil).Fs)) +} + +// Hash mocks base method. +func (m *MockRCloneObject) Hash(ctx context.Context, ty hash.Type) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Hash", ctx, ty) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Hash indicates an expected call of Hash. +func (mr *MockRCloneObjectMockRecorder) Hash(ctx, ty any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Hash", reflect.TypeOf((*MockRCloneObject)(nil).Hash), ctx, ty) +} + +// ModTime mocks base method. +func (m *MockRCloneObject) ModTime(arg0 context.Context) time.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ModTime", arg0) + ret0, _ := ret[0].(time.Time) + return ret0 +} + +// ModTime indicates an expected call of ModTime. +func (mr *MockRCloneObjectMockRecorder) ModTime(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ModTime", reflect.TypeOf((*MockRCloneObject)(nil).ModTime), arg0) +} + +// Open mocks base method. +func (m *MockRCloneObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + m.ctrl.T.Helper() + varargs := []any{ctx} + for _, a := range options { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Open", varargs...) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Open indicates an expected call of Open. +func (mr *MockRCloneObjectMockRecorder) Open(ctx any, options ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx}, options...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Open", reflect.TypeOf((*MockRCloneObject)(nil).Open), varargs...) +} + +// Remote mocks base method. +func (m *MockRCloneObject) Remote() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remote") + ret0, _ := ret[0].(string) + return ret0 +} + +// Remote indicates an expected call of Remote. +func (mr *MockRCloneObjectMockRecorder) Remote() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remote", reflect.TypeOf((*MockRCloneObject)(nil).Remote)) +} + +// Remove mocks base method. +func (m *MockRCloneObject) Remove(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Remove indicates an expected call of Remove. +func (mr *MockRCloneObjectMockRecorder) Remove(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockRCloneObject)(nil).Remove), ctx) +} + +// SetModTime mocks base method. +func (m *MockRCloneObject) SetModTime(ctx context.Context, t time.Time) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetModTime", ctx, t) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetModTime indicates an expected call of SetModTime. +func (mr *MockRCloneObjectMockRecorder) SetModTime(ctx, t any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetModTime", reflect.TypeOf((*MockRCloneObject)(nil).SetModTime), ctx, t) +} + +// Size mocks base method. +func (m *MockRCloneObject) Size() int64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Size") + ret0, _ := ret[0].(int64) + return ret0 +} + +// Size indicates an expected call of Size. +func (mr *MockRCloneObjectMockRecorder) Size() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockRCloneObject)(nil).Size)) +} + +// Storable mocks base method. +func (m *MockRCloneObject) Storable() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Storable") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Storable indicates an expected call of Storable. +func (mr *MockRCloneObjectMockRecorder) Storable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Storable", reflect.TypeOf((*MockRCloneObject)(nil).Storable)) +} + +// String mocks base method. +func (m *MockRCloneObject) String() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String") + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockRCloneObjectMockRecorder) String() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockRCloneObject)(nil).String)) +} + +// Update mocks base method. +func (m *MockRCloneObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + m.ctrl.T.Helper() + varargs := []any{ctx, in, src} + for _, a := range options { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockRCloneObjectMockRecorder) Update(ctx, in, src any, options ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in, src}, options...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRCloneObject)(nil).Update), varargs...) +} + +// MockRCloneInfo is a mock of RCloneInfo interface. +type MockRCloneInfo struct { + ctrl *gomock.Controller + recorder *MockRCloneInfoMockRecorder +} + +// MockRCloneInfoMockRecorder is the mock recorder for MockRCloneInfo. +type MockRCloneInfoMockRecorder struct { + mock *MockRCloneInfo +} + +// NewMockRCloneInfo creates a new mock instance. +func NewMockRCloneInfo(ctrl *gomock.Controller) *MockRCloneInfo { + mock := &MockRCloneInfo{ctrl: ctrl} + mock.recorder = &MockRCloneInfoMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRCloneInfo) EXPECT() *MockRCloneInfoMockRecorder { + return m.recorder +} + +// Features mocks base method. +func (m *MockRCloneInfo) Features() *fs.Features { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Features") + ret0, _ := ret[0].(*fs.Features) + return ret0 +} + +// Features indicates an expected call of Features. +func (mr *MockRCloneInfoMockRecorder) Features() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Features", reflect.TypeOf((*MockRCloneInfo)(nil).Features)) +} + +// Hashes mocks base method. +func (m *MockRCloneInfo) Hashes() hash.Set { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Hashes") + ret0, _ := ret[0].(hash.Set) + return ret0 +} + +// Hashes indicates an expected call of Hashes. +func (mr *MockRCloneInfoMockRecorder) Hashes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Hashes", reflect.TypeOf((*MockRCloneInfo)(nil).Hashes)) +} + +// Name mocks base method. +func (m *MockRCloneInfo) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockRCloneInfoMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockRCloneInfo)(nil).Name)) +} + +// Precision mocks base method. +func (m *MockRCloneInfo) Precision() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Precision") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// Precision indicates an expected call of Precision. +func (mr *MockRCloneInfoMockRecorder) Precision() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Precision", reflect.TypeOf((*MockRCloneInfo)(nil).Precision)) +} + +// Root mocks base method. +func (m *MockRCloneInfo) Root() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Root") + ret0, _ := ret[0].(string) + return ret0 +} + +// Root indicates an expected call of Root. +func (mr *MockRCloneInfoMockRecorder) Root() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockRCloneInfo)(nil).Root)) +} + +// String mocks base method. +func (m *MockRCloneInfo) String() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "String") + ret0, _ := ret[0].(string) + return ret0 +} + +// String indicates an expected call of String. +func (mr *MockRCloneInfoMockRecorder) String() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "String", reflect.TypeOf((*MockRCloneInfo)(nil).String)) +} diff --git a/internal/vendors/supermicro/supermicro.go b/internal/vendors/supermicro/supermicro.go index b8500f7e..6fe6f755 100644 --- a/internal/vendors/supermicro/supermicro.go +++ b/internal/vendors/supermicro/supermicro.go @@ -6,156 +6,65 @@ import ( "fmt" "io" "net/http" - "os" "strings" "time" - "github.com/metal-toolbox/firmware-syncer/internal/config" - "github.com/metal-toolbox/firmware-syncer/internal/inventory" - "github.com/metal-toolbox/firmware-syncer/internal/vendors" "github.com/pkg/errors" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/operations" "github.com/sirupsen/logrus" + "github.com/metal-toolbox/firmware-syncer/internal/vendors" + serverservice "go.hollow.sh/serverservice/pkg/api/v1" ) -type Supermicro struct { - firmwares []*serverservice.ComponentFirmwareVersion - logger *logrus.Logger - metrics *vendors.Metrics - inventory *inventory.ServerService - dstCfg *config.S3Bucket - dstFs fs.Fs - tmpFs fs.Fs -} - -func New(ctx context.Context, firmwares []*serverservice.ComponentFirmwareVersion, cfg *config.Configuration, logger *logrus.Logger) (vendors.Vendor, error) { - // init inventory - i, err := inventory.New(ctx, cfg.ServerserviceOptions, cfg.ArtifactsURL, logger) - if err != nil { - return nil, err - } - - vendors.SetRcloneLogging(logger) - - dstFs, err := vendors.InitS3Fs(ctx, cfg.FirmwareRepository, "/") - if err != nil { - return nil, err - } - - tmpFs, err := vendors.InitLocalFs(ctx, &vendors.LocalFsConfig{Root: "/tmp"}) - if err != nil { - return nil, err - } +var ErrMissingFirmwareID = errors.New("upstream URL is missing firmwareID") - return &Supermicro{ - firmwares: firmwares, - logger: logger, - metrics: vendors.NewMetrics(), - inventory: i, - dstCfg: cfg.FirmwareRepository, - dstFs: dstFs, - tmpFs: tmpFs, - }, nil +type Downloader struct { + logger *logrus.Logger } -func (s *Supermicro) Stats() *vendors.Metrics { - return s.metrics +// NewSupermicroDownloader creates a new Downloader for downloading files from Supermicro. +func NewSupermicroDownloader(logger *logrus.Logger) vendors.Downloader { + return &Downloader{logger: logger} } -func (s *Supermicro) Sync(ctx context.Context) error { - for _, fw := range s.firmwares { - fwID := strings.Split(fw.UpstreamURL, "=")[1] +// Download will download a file for the given firmware to the given downloadDir, +// and will return the full path to the downloaded file. +func (d *Downloader) Download(ctx context.Context, downloadDir string, firmware *serverservice.ComponentFirmwareVersion) (string, error) { + urlSplit := strings.Split(firmware.UpstreamURL, "=") - archiveURL, archiveChecksum, err := getArchiveURLAndChecksum(ctx, fwID) - - s.logger.WithFields( - logrus.Fields{ - "archiveURL": archiveURL, - "archiveChecksum": archiveChecksum, - }, - ).Debug("found archive") - - if err != nil { - s.logger.WithFields( - logrus.Fields{ - "fwID": fwID, - }, - ).Debug("failed to get archiveURL and archiveChecksum") - - return err - } - - // In case the file already exists in dst, don't copy it - if exists, _ := fs.FileExists(ctx, s.dstFs, vendors.DstPath(fw)); exists { - s.logger.WithFields( - logrus.Fields{ - "filename": fw.Filename, - }, - ).Debug("firmware already exists at dst") - - continue - } - - // initialize a tmpDir so we can download and unpack the zip archive - tmpDir, err := os.MkdirTemp(s.tmpFs.Root(), "firmware-archive") - if err != nil { - return err - } - - s.logger.Debug("Downloading archive") + if len(urlSplit) < 2 { + return "", errors.Wrap(ErrMissingFirmwareID, firmware.UpstreamURL) + } - archivePath, err := vendors.DownloadFirmwareArchive(ctx, tmpDir, archiveURL, archiveChecksum) - if err != nil { - return err - } + firmwareID := urlSplit[1] + archiveURL, archiveChecksum, err := getArchiveURLAndChecksum(ctx, firmwareID) - s.logger.WithFields( - logrus.Fields{ - "archivePath": archivePath, - }, - ).Debug("Archive downloaded.") + d.logger.WithField("archiveURL", archiveURL). + WithField("archiveChecksum", archiveChecksum). + Debug("found archive") - s.logger.Debug("Extracting firmware from archive") + if err != nil { + d.logger.WithField("firmwareID", firmwareID).Debug("failed to get archiveURL and archiveChecksum") + return "", err + } - fwFile, err := vendors.ExtractFromZipArchive(archivePath, fw.Filename, fw.Checksum) - if err != nil { - return err - } + d.logger.Debug("Downloading archive") - s.logger.WithFields( - logrus.Fields{ - "fwFile": fwFile.Name(), - }, - ).Debug("Firmware extracted.") - - s.logger.WithFields( - logrus.Fields{ - "src": fwFile.Name(), - "dst": vendors.DstPath(fw), - }, - ).Info("Sync Supermicro") - - // Remove root of tmpdir from filename since CopyFile doesn't use it - tmpFwPath := strings.Replace(fwFile.Name(), s.tmpFs.Root(), "", 1) - - err = operations.CopyFile(ctx, s.dstFs, s.tmpFs, vendors.DstPath(fw), tmpFwPath) - if err != nil { - return err - } + archivePath, err := vendors.DownloadFirmwareArchive(ctx, downloadDir, archiveURL, archiveChecksum) + if err != nil { + return "", err + } - // Clean up tmpDir after copying the extracted firmware to dst. - os.RemoveAll(tmpDir) + d.logger.WithField("archivePath", archivePath).Debug("Archive downloaded.") + d.logger.Debug("Extracting firmware from archive") - err = s.inventory.Publish(ctx, fw) - if err != nil { - return err - } + fwFile, err := vendors.ExtractFromZipArchive(archivePath, firmware.Filename, "") + if err != nil { + return "", err } - return nil + return fwFile.Name(), nil } func getArchiveURLAndChecksum(ctx context.Context, id string) (url, checksum string, err error) { diff --git a/internal/vendors/syncer.go b/internal/vendors/syncer.go new file mode 100644 index 00000000..fc6b6679 --- /dev/null +++ b/internal/vendors/syncer.go @@ -0,0 +1,124 @@ +package vendors + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/operations" + "github.com/sirupsen/logrus" + + "github.com/metal-toolbox/firmware-syncer/internal/inventory" + + serverservice "go.hollow.sh/serverservice/pkg/api/v1" +) + +type Syncer struct { + dstFs fs.Fs + tmpFs fs.Fs + downloader Downloader + firmwares []*serverservice.ComponentFirmwareVersion + logger *logrus.Logger + inventory inventory.ServerService +} + +// NewSyncer creates a new Syncer. +func NewSyncer( + dstFs fs.Fs, + tmpFs fs.Fs, + downloader Downloader, + inventoryClient inventory.ServerService, + firmwares []*serverservice.ComponentFirmwareVersion, + logger *logrus.Logger, +) Vendor { + SetRcloneLogging(logger) + + return &Syncer{ + dstFs: dstFs, + tmpFs: tmpFs, + downloader: downloader, + inventory: inventoryClient, + firmwares: firmwares, + logger: logger, + } +} + +// Sync will synchronize the firmwares with the destination file system and inventory. +// Files that do not exist on the destination will be downloaded from their source and uploaded to the destination. +// Information about the firmware file will be updated using the inventory client. +func (s *Syncer) Sync(ctx context.Context) (err error) { + for _, firmware := range s.firmwares { + if err = s.syncFirmware(ctx, firmware); err != nil { + msg := fmt.Sprintf("failed to sync firmware %s", firmware.Filename) + return errors.Wrap(err, msg) + } + } + + return nil +} + +// syncFirmware does the synchronization for the given firmware. +func (s *Syncer) syncFirmware(ctx context.Context, firmware *serverservice.ComponentFirmwareVersion) error { + destPath := DstPath(firmware) + + logMsg := s.logger.WithField("firmware", firmware.Filename). + WithField("vendor", firmware.Vendor). + WithField("url", firmware.UpstreamURL) + + logMsg.Info("Syncing Firmware") + + fileExists, err := fs.FileExists(ctx, s.dstFs, destPath) + if err != nil { + return errors.Wrap(err, "failure checking if firmware file exists") + } + + if !fileExists { + downloadDir, err := os.MkdirTemp(s.tmpFs.Root(), "firmware-download") + if err != nil { + return errors.Wrap(err, "failure creating download directory") + } + + defer func() { + if err = os.RemoveAll(downloadDir); err != nil { + logMsg.WithError(err).Error("Failure to clean up download directory") + } + }() + + firmwareFilePath, err := s.downloader.Download(ctx, downloadDir, firmware) + if err != nil { + logMsg.WithError(err).Error("Failed to download firmware") + return nil // Only logging the error, so we don't fail the whole process + } + + if err = validateChecksum(firmwareFilePath, firmware.Checksum); err != nil { + logMsg.WithError(err).Error("Checksum validation failure") + return nil // Only logging the error, so we don't fail the whole process + } + + if err = s.uploadFile(ctx, firmwareFilePath, destPath); err != nil { + msg := fmt.Sprintf("failure to upload firmware %s", firmware.Filename) + return errors.Wrap(err, msg) + } + } + + return s.inventory.Publish(ctx, firmware) +} + +func (s *Syncer) uploadFile(ctx context.Context, firmwarePath, destPath string) error { + // Remove root of tmpdir from filename since CopyFile doesn't use it + firmwareRelativePath := strings.Replace(firmwarePath, s.tmpFs.Root(), "", 1) + + return operations.CopyFile(ctx, s.dstFs, s.tmpFs, destPath, firmwareRelativePath) +} + +func validateChecksum(file, checksum string) error { + if !ValidateChecksum(file, checksum) { + msg := fmt.Sprintf("Checksum validation failed: %s, expected checksum: %s", file, checksum) + return errors.Wrap(ErrChecksumValidate, msg) + } + + return nil +} diff --git a/internal/vendors/syncer_test.go b/internal/vendors/syncer_test.go new file mode 100644 index 00000000..22f3b820 --- /dev/null +++ b/internal/vendors/syncer_test.go @@ -0,0 +1,154 @@ +package vendors + +import ( + "context" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/rclone/rclone/fs" + "github.com/stretchr/testify/assert" + serverservice "go.hollow.sh/serverservice/pkg/api/v1" + "go.uber.org/mock/gomock" + + mockinventory "github.com/metal-toolbox/firmware-syncer/internal/inventory/mocks" + "github.com/metal-toolbox/firmware-syncer/internal/logging" + mockvendors "github.com/metal-toolbox/firmware-syncer/internal/vendors/mocks" +) + +//go:generate mockgen -source=syncer_test.go -destination=mocks/rclone.go RCloneFS + +// RCloneFS interface is just to help generate the mocks +type RCloneFS interface { + fs.Fs +} + +//go:generate mockgen -source=syncer_test.go -destination=mocks/rclone.go RCloneObject + +// RCloneObject interface is just to help generate the mocks +type RCloneObject interface { + fs.Object +} + +//go:generate mockgen -source=syncer_test.go -destination=mocks/rclone.go RCloneInfo + +// RCloneInfo interface is just to help generate the mocks +type RCloneInfo interface { + fs.Info +} + +type rootDirMatcher struct { + root string +} + +func (t *rootDirMatcher) Matches(x interface{}) bool { + tempDir, ok := x.(string) + if !ok { + return false + } + + return strings.HasPrefix(tempDir, t.root) +} + +func (t *rootDirMatcher) String() string { + return fmt.Sprintf("Does not have root dir %s", t.root) +} + +func MatchesRootDir(root string) gomock.Matcher { + return &rootDirMatcher{root: root} +} + +func TestSyncer(t *testing.T) { + logger := logging.NewLogger("debug") + ctx := context.Background() + tmpDir := os.TempDir() + + firmware := &serverservice.ComponentFirmwareVersion{ + UUID: uuid.New(), + Vendor: "foo-vendor", + Filename: "foobar1.zip", + Version: "v0.0.0", + Component: "foo-component", + Checksum: "79ec3cf629b56317111d5640b8df1220", // real checksum of fixtures/foobar1.zip + UpstreamURL: "vendor-url", + RepositoryURL: "repository-url", + } + + firmwares := []*serverservice.ComponentFirmwareVersion{ + firmware, + } + + tests := []struct { + name string + fileShouldExist bool + wantErr assert.ErrorAssertionFunc + }{ + { + name: "file does not exist", + fileShouldExist: false, + wantErr: assert.NoError, + }, + { + name: "file already exists", + fileShouldExist: true, + wantErr: assert.NoError, + }, + } + + localPath := path.Join("fixtures", firmware.Filename) + dstPath := path.Join(firmware.Vendor, firmware.Filename) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + + mockDstFs := mockvendors.NewMockRCloneFS(ctrl) + mockTmpFs := mockvendors.NewMockRCloneFS(ctrl) + mockDownloader := mockvendors.NewMockDownloader(ctrl) + obj := mockvendors.NewMockRCloneObject(ctrl) + + if !tt.fileShouldExist { + mockDownloader.EXPECT(). + Download(ctx, MatchesRootDir(tmpDir), firmware). + Return(localPath, nil) + + mockDstFs.EXPECT().NewObject(ctx, dstPath).Return(nil, fs.ErrorObjectNotFound) + + info := mockvendors.NewMockRCloneInfo(ctrl) + info.EXPECT().Precision().Return(time.Duration(0)).AnyTimes() + + obj.EXPECT().Size().Return(int64(0)).AnyTimes() + obj.EXPECT().ModTime(ctx).Return(time.Now()).AnyTimes() + obj.EXPECT().Fs().Return(info).AnyTimes() + obj.EXPECT().String().Return("rclone-object").AnyTimes() + + mockDstFs.EXPECT().Root() + mockDstFs.EXPECT().Name() + + mockTmpFs.EXPECT().NewObject(ctx, localPath).Return(obj, nil) + mockTmpFs.EXPECT().Root().Return(tmpDir).AnyTimes() + mockTmpFs.EXPECT().Name().Return("local").AnyTimes() + } + + mockDstFs.EXPECT().NewObject(ctx, dstPath).Return(obj, nil).AnyTimes() + + mockInventory := mockinventory.NewMockServerService(ctrl) + mockInventory.EXPECT().Publish(ctx, firmware) + + s := NewSyncer( + mockDstFs, + mockTmpFs, + mockDownloader, + mockInventory, + firmwares, + logger, + ) + + tt.wantErr(t, s.Sync(ctx), tt.name, "Syncer.Sync") + }) + } +} diff --git a/internal/vendors/vendors.go b/internal/vendors/vendors.go index b0c8f6ef..17ca700c 100644 --- a/internal/vendors/vendors.go +++ b/internal/vendors/vendors.go @@ -16,7 +16,6 @@ const ( type Vendor interface { Sync(ctx context.Context) error - Stats() *Metrics } // Metrics is a struct with a key value map under an RWMutex