diff --git a/cmd/common.go b/cmd/common.go index 3f5792d..4dd1b95 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -42,6 +42,12 @@ func SetViperDefaults() { "http://artifactory.infra.corp.arista.io") viper.SetDefault("DnfConfigFile", "/usr/share/eext/dnfconfig.yaml") + viper.SetDefault("SrcRepoHost", + "http://artifactory.infra.corp.arista.io") + viper.SetDefault("SrcConfigFile", + "/usr/share/eext/srcconfig.yaml") + viper.SetDefault("SrcRepoPathPrefix", + "artifactory/eext-sources") viper.SetDefault("PkiPath", "/etc/pki/eext") viper.SetDefault("SrcEnvPrefix", diff --git a/cmd/create_srpm_test.go b/cmd/create_srpm_test.go index 9bfabcc..4edb292 100644 --- a/cmd/create_srpm_test.go +++ b/cmd/create_srpm_test.go @@ -35,11 +35,15 @@ func testCreateSrpm(t *testing.T, t.Logf("DestDir: %s", destDir) testutil.SetupViperConfig( - "", // srcDir - workingDir, destDir, - "", // depsDir - "", // repoHost, - "", // dnfConfigFile + "", // srcDir + workingDir, // workingDir + destDir, // destDir + "", // depsDir + "", // repoHost, + "", // dnfConfigFile + "", // srcRepoHost + "testData/configfiles/srcconfig.yaml", // srcConfigFile + "", // srcRepoPathPrefix ) defer viper.Reset() if sources != nil { diff --git a/cmd/mock_test.go b/cmd/mock_test.go index bc7941f..b5ad034 100644 --- a/cmd/mock_test.go +++ b/cmd/mock_test.go @@ -69,6 +69,9 @@ func testMock(t *testing.T, setupSrcEnv bool) { "", // depsDir "", // repoHost, "", // dnfConfigFile + "", // srcRepoHost + "", // srcConfigFile + "", // srcRepoPathPrefix ) args := []string{"create-srpm", "--repo", repoName} diff --git a/cmd/testData/configfiles/srcconfig.yaml b/cmd/testData/configfiles/srcconfig.yaml new file mode 100644 index 0000000..aeff6b7 --- /dev/null +++ b/cmd/testData/configfiles/srcconfig.yaml @@ -0,0 +1,5 @@ +--- +# yamllint disable rule:line-length +source-bundle: + srpm: + url-format: "{{.Host}}/{{.PathPrefix}}/eext-testData/{{.PkgName}}-{{.Version}}.src.rpm" diff --git a/cmd/testData/debugedit-1/eext.yaml b/cmd/testData/debugedit-1/eext.yaml index a5ab2a4..13c1f64 100644 --- a/cmd/testData/debugedit-1/eext.yaml +++ b/cmd/testData/debugedit-1/eext.yaml @@ -3,7 +3,7 @@ package: - name: debugedit upstream-sources: - - source: https://artifactory.infra.corp.arista.io/artifactory/eext-sources/eext-testData/debugedit-5.0-3.el9.src.rpm + - full-url: "{{.Host}}/{{.PathPrefix}}/eext-testData/debugedit-5.0-3.el9.src.rpm" type: srpm build: repo-bundle: diff --git a/cmd/testData/debugedit-1/spec/debugedit.spec b/cmd/testData/debugedit-1/spec/debugedit.spec index c89693b..168a0bf 100644 --- a/cmd/testData/debugedit-1/spec/debugedit.spec +++ b/cmd/testData/debugedit-1/spec/debugedit.spec @@ -4,7 +4,7 @@ Name: debugedit Version: 5.0 -Release: %{?release:%{release}}%{!?release:eng} +Release: %{?eext_release:%{eext_release}}%{!?eext_release:eng} Summary: Tools for debuginfo creation License: GPLv3+ and GPLv2+ and LGPLv2+ URL: https://sourceware.org/debugedit/ diff --git a/cmd/testData/debugedit-2/eext.yaml b/cmd/testData/debugedit-2/eext.yaml index 6274e60..bd274a2 100644 --- a/cmd/testData/debugedit-2/eext.yaml +++ b/cmd/testData/debugedit-2/eext.yaml @@ -3,7 +3,10 @@ package: - name: debugedit upstream-sources: - - source: https://artifactory.infra.corp.arista.io/artifactory/eext-sources/eext-testData/debugedit-5.0-3.el9.src.rpm + - source-bundle: + name: srpm + override: + version: 5.0-3.el9 type: unmodified-srpm build: repo-bundle: diff --git a/cmd/testData/mrtparse-1/eext.yaml b/cmd/testData/mrtparse-1/eext.yaml index 6ab1696..9909b0c 100644 --- a/cmd/testData/mrtparse-1/eext.yaml +++ b/cmd/testData/mrtparse-1/eext.yaml @@ -3,10 +3,11 @@ package: - name: mrtparse upstream-sources: - - source: file:///mrtparse-2.0.1.tar.gz + - full-url: file:///mrtparse-2.0.1.tar.gz signature: - detached-sig: file:///mrtparse-2.0.1.tar.gz.sig - public-key: mrtparse/mrtparsePubKey.pem + detached-sig: + full-url: file:///mrtparse-2.0.1.tar.gz.sig + public-key: mrtparse/mrtparsePubKey.pem type: tarball build: include: diff --git a/cmd/testData/mrtparse-1/spec/mrtparse.spec b/cmd/testData/mrtparse-1/spec/mrtparse.spec index 1312cac..c581380 100644 --- a/cmd/testData/mrtparse-1/spec/mrtparse.spec +++ b/cmd/testData/mrtparse-1/spec/mrtparse.spec @@ -5,17 +5,17 @@ Patch1000: arista-remove-default-sigpipe.patch Summary: Tool for parsing routing information dump files in MRT format Name: mrtparse Version: 2.0.1 -Release: %{?release:%{release}}%{!?release:eng} +Release: %{?eext_release:%{eext_release}}%{!?eext_release:eng} -License: Apache +License: Apache Source0: mrtparse-2.0.1.tar.gz Url: https://github.com/YoshiyukiYamauchi/mrtparse BuildArch: noarch %description -mrtprse is a module to read and analyze the MRT format data. The MRT format data -can be used to export routing protocol messages, state changes, and routing -information base contents, and is standardized in RFC6396. Programs like Quagga +mrtprse is a module to read and analyze the MRT format data. The MRT format data +can be used to export routing protocol messages, state changes, and routing +information base contents, and is standardized in RFC6396. Programs like Quagga / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data. %package -n python3-mrtparse @@ -25,9 +25,9 @@ BuildRequires: python3-setuptools %{?python_provide:%python_provide python3-mrtparse} %description -n python3-mrtparse -mrtprse is a module to read and analyze the MRT format data. The MRT format data -can be used to export routing protocol messages, state changes, and routing -information base contents, and is standardized in RFC6396. Programs like Quagga +mrtprse is a module to read and analyze the MRT format data. The MRT format data +can be used to export routing protocol messages, state changes, and routing +information base contents, and is standardized in RFC6396. Programs like Quagga / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data. %prep diff --git a/cmd/testData/mrtparse-2/eext.yaml b/cmd/testData/mrtparse-2/eext.yaml index fdbc1ff..cc452a2 100644 --- a/cmd/testData/mrtparse-2/eext.yaml +++ b/cmd/testData/mrtparse-2/eext.yaml @@ -4,7 +4,7 @@ package: - name: mrtparse subdir: true upstream-sources: - - source: file:///mrtparse-2.0.1.tar.gz + - full-url: file:///mrtparse-2.0.1.tar.gz signature: skip-check: true type: tarball diff --git a/cmd/testData/mrtparse-2/mrtparse/spec/mrtparse.spec b/cmd/testData/mrtparse-2/mrtparse/spec/mrtparse.spec index 1312cac..c581380 100644 --- a/cmd/testData/mrtparse-2/mrtparse/spec/mrtparse.spec +++ b/cmd/testData/mrtparse-2/mrtparse/spec/mrtparse.spec @@ -5,17 +5,17 @@ Patch1000: arista-remove-default-sigpipe.patch Summary: Tool for parsing routing information dump files in MRT format Name: mrtparse Version: 2.0.1 -Release: %{?release:%{release}}%{!?release:eng} +Release: %{?eext_release:%{eext_release}}%{!?eext_release:eng} -License: Apache +License: Apache Source0: mrtparse-2.0.1.tar.gz Url: https://github.com/YoshiyukiYamauchi/mrtparse BuildArch: noarch %description -mrtprse is a module to read and analyze the MRT format data. The MRT format data -can be used to export routing protocol messages, state changes, and routing -information base contents, and is standardized in RFC6396. Programs like Quagga +mrtprse is a module to read and analyze the MRT format data. The MRT format data +can be used to export routing protocol messages, state changes, and routing +information base contents, and is standardized in RFC6396. Programs like Quagga / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data. %package -n python3-mrtparse @@ -25,9 +25,9 @@ BuildRequires: python3-setuptools %{?python_provide:%python_provide python3-mrtparse} %description -n python3-mrtparse -mrtprse is a module to read and analyze the MRT format data. The MRT format data -can be used to export routing protocol messages, state changes, and routing -information base contents, and is standardized in RFC6396. Programs like Quagga +mrtprse is a module to read and analyze the MRT format data. The MRT format data +can be used to export routing protocol messages, state changes, and routing +information base contents, and is standardized in RFC6396. Programs like Quagga / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data. %prep diff --git a/configfiles/srcconfig.yaml b/configfiles/srcconfig.yaml new file mode 100644 index 0000000..1fc3215 --- /dev/null +++ b/configfiles/srcconfig.yaml @@ -0,0 +1,10 @@ +--- +# yamllint disable rule:line-length +source-bundle: + srpm: + url-format: "{{.Host}}/{{.PathPrefix}}/source-bundles/srpm/{{.PkgName}}/{{.PkgName}}-{{.Version}}.src.rpm" + tarball: + url-format: "{{.Host}}/{{.PathPrefix}}/source-bundles/tarball/{{.PkgName}}/{{.Version}}/{{.PkgName}}-{{.Version}}{{.Suffix}}" + default-src-suffix: .tar.gz + default-sig-suffix: .sig + has-detached-sig: true diff --git a/impl/checkenv.go b/impl/checkenv.go index 424a766..f3a43f0 100644 --- a/impl/checkenv.go +++ b/impl/checkenv.go @@ -21,6 +21,7 @@ func CheckEnv() error { destDir := viper.GetString("DestDir") mockCfgTemplate := viper.GetString("MockCfgTemplate") dnfConfigFile := viper.GetString("DnfConfigFile") + srcConfigFile := viper.GetString("SrcConfigFile") pkiPath := viper.GetString("PkiPath") var aggError string @@ -65,6 +66,11 @@ func CheckEnv() error { failed = true } + if err := util.CheckPath(srcConfigFile, false, false); err != nil { + aggError += fmt.Sprintf("\ntrouble with SrcConfigFile: %s", err) + failed = true + } + if err := util.CheckPath(pkiPath, true, false); err != nil { aggError += fmt.Sprintf("\ntrouble with PkiPath: %s", err) failed = true diff --git a/impl/create_srpm.go b/impl/create_srpm.go index a5fb194..80b6e2b 100644 --- a/impl/create_srpm.go +++ b/impl/create_srpm.go @@ -13,6 +13,7 @@ import ( "golang.org/x/exp/slices" "code.arista.io/eos/tools/eext/manifest" + "code.arista.io/eos/tools/eext/srcconfig" "code.arista.io/eos/tools/eext/util" ) @@ -30,6 +31,7 @@ type srpmBuilder struct { errPrefixBase util.ErrPrefix errPrefix util.ErrPrefix upstreamSrc []upstreamSrcSpec + srcConfig *srcconfig.SrcConfig } // CreateSrpmExtraCmdlineArgs is a bundle of extra args for impl.CreateSrpm @@ -86,18 +88,27 @@ func (bldr *srpmBuilder) fetchUpstream() error { } for _, upstreamSrcFromManifest := range bldr.pkgSpec.UpstreamSrc { - if upstreamSrcFromManifest.Source == "" { - return fmt.Errorf("%ssource not specified for upstream-sources entry", - bldr.errPrefix) + srcParams, err := srcconfig.GetSrcParams( + bldr.pkgSpec.Name, + upstreamSrcFromManifest.FullURL, + upstreamSrcFromManifest.SourceBundle.Name, + upstreamSrcFromManifest.Signature.DetachedSignature.FullURL, + upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride, + upstreamSrcFromManifest.Signature.DetachedSignature.OnUncompressed, + bldr.srcConfig, + bldr.errPrefix) + if err != nil { + return fmt.Errorf("%sUnable to get source params for %s", + err, upstreamSrcFromManifest.SourceBundle.Name) } var downloadErr error upstreamSrc := upstreamSrcSpec{} - bldr.log("downloading %s", upstreamSrcFromManifest.Source) + bldr.log("downloading %s", srcParams.SrcURL) // Download source if upstreamSrc.sourceFile, downloadErr = download( - upstreamSrcFromManifest.Source, + srcParams.SrcURL, downloadDir, repo, pkg, isPkgSubdirInRepo, bldr.errPrefix); downloadErr != nil { @@ -105,23 +116,23 @@ func (bldr *srpmBuilder) fetchUpstream() error { } bldr.log("downloaded") - signatureSpec := upstreamSrcFromManifest.Signature - upstreamSrc.skipSigCheck = signatureSpec.SkipCheck + upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck + pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey - if bldr.pkgSpec.Type == "tarball" && !signatureSpec.SkipCheck { - if signatureSpec.DetachedSig == "" || signatureSpec.PubKey == "" { + if bldr.pkgSpec.Type == "tarball" && !upstreamSrc.skipSigCheck { + if srcParams.SignatureURL == "" || pubKey == "" { return fmt.Errorf("%sNo detached-signature/public-key specified for upstream-sources entry %s", - bldr.errPrefix, upstreamSrcFromManifest.Source) + bldr.errPrefix, srcParams.SrcURL) } if upstreamSrc.sigFile, downloadErr = download( - signatureSpec.DetachedSig, + srcParams.SignatureURL, downloadDir, repo, pkg, isPkgSubdirInRepo, bldr.errPrefix); downloadErr != nil { return downloadErr } - pubKeyPath := filepath.Join(getDetachedSigDir(), signatureSpec.PubKey) + pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { return fmt.Errorf("%sCannot find public-key at path %s", bldr.errPrefix, pubKeyPath) @@ -130,11 +141,11 @@ func (bldr *srpmBuilder) fetchUpstream() error { } else if bldr.pkgSpec.Type == "srpm" || bldr.pkgSpec.Type == "unmodified-srpm" { // We don't expect SRPMs to have detached signature or // to be validated with a public-key specified in manifest. - if signatureSpec.DetachedSig != "" { + if srcParams.SignatureURL != "" { return fmt.Errorf("%sUnexpected detached-sig specified for SRPM", bldr.errPrefix) } - if signatureSpec.PubKey != "" { + if pubKey != "" { return fmt.Errorf("%sUnexpected public-key specified for SRPM", bldr.errPrefix) } @@ -381,7 +392,7 @@ func (bldr *srpmBuilder) build(prep bool) error { if rpmReleaseMacro != "" { rpmbuildArgs = append(rpmbuildArgs, []string{ - "--define", fmt.Sprintf("release %s", rpmReleaseMacro), + "--define", fmt.Sprintf("eext_release %s", rpmReleaseMacro), }...) } rpmbuildArgs = append(rpmbuildArgs, specFile) @@ -527,6 +538,11 @@ func CreateSrpm(repo string, pkg string, extraArgs CreateSrpmExtraCmdlineArgs) e return loadManifestErr } + srcConfig, err := srcconfig.LoadSrcConfig() + if err != nil { + return err + } + var pkgSpecified bool = (pkg != "") found := !pkgSpecified for _, pkgSpec := range repoManifest.Package { @@ -541,6 +557,7 @@ func CreateSrpm(repo string, pkg string, extraArgs CreateSrpmExtraCmdlineArgs) e repo: repo, skipBuildPrep: extraArgs.SkipBuildPrep, errPrefixBase: errPrefixBase, + srcConfig: srcConfig, } bldr.setupStageErrPrefix("") diff --git a/impl/mock_cfg.go b/impl/mock_cfg.go index 426151a..1684e08 100644 --- a/impl/mock_cfg.go +++ b/impl/mock_cfg.go @@ -58,7 +58,7 @@ func (cfgBldr *mockCfgBuilder) populateTemplateData() error { cfgBldr.templateData.Macros = make(map[string]string) if cfgBldr.rpmReleaseMacro != "" { - cfgBldr.templateData.Macros["release"] = cfgBldr.rpmReleaseMacro + cfgBldr.templateData.Macros["eext_release"] = cfgBldr.rpmReleaseMacro } if cfgBldr.eextSignature != "" { diff --git a/impl/mock_cfg_test.go b/impl/mock_cfg_test.go index 0b1c75b..2391543 100644 --- a/impl/mock_cfg_test.go +++ b/impl/mock_cfg_test.go @@ -45,7 +45,10 @@ func testMockConfig(t *testing.T, chained bool) { destDir, "", // depsDir "https://foo.org", // repoHost - "testData/dnfconfig.yaml", //dnfConfigFile + "testData/dnfconfig.yaml", // dnfConfigFile + "", // srcRepoHost + "", // srcConfigFile + "", // srcRepoPathPrefix ) defer viper.Reset() diff --git a/impl/setupDeps_test.go b/impl/setupDeps_test.go index bb9ff80..59db1b1 100644 --- a/impl/setupDeps_test.go +++ b/impl/setupDeps_test.go @@ -49,6 +49,9 @@ func TestSetupDeps(t *testing.T) { depsDir, "", // repoHost "", // dnf config file + "", // src repo host + "", // src config file + "", // src repo path prefix ) defer viper.Reset() diff --git a/impl/testData/expected-mock.cfg b/impl/testData/expected-mock.cfg index c70fa06..d0b3d0d 100644 --- a/impl/testData/expected-mock.cfg +++ b/impl/testData/expected-mock.cfg @@ -18,7 +18,7 @@ config_opts['target_arch'] = "x86_64" # Autogenerated macros config_opts['macros']['distribution'] = 'eextsig=my-signature' -config_opts['macros']['release'] = 'my-release' +config_opts['macros']['eext_release'] = 'my-release' # Autogenerated dnf.conf config_opts['dnf.conf'] = """ diff --git a/impl/testData/manifest.yaml b/impl/testData/manifest.yaml index 0a54fa0..37c4273 100644 --- a/impl/testData/manifest.yaml +++ b/impl/testData/manifest.yaml @@ -2,7 +2,7 @@ package: - name: pkg1 upstream-sources: - - source: http://foo.org/pkg1.src.rpm + - full-url: http://foo.org/pkg1.src.rpm type: srpm build: repo-bundle: diff --git a/manifest/manifest.go b/manifest/manifest.go index e654ea8..aecacf4 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -12,6 +12,7 @@ import ( "gopkg.in/yaml.v2" "code.arista.io/eos/tools/eext/dnfconfig" + "code.arista.io/eos/tools/eext/srcconfig" "code.arista.io/eos/tools/eext/util" ) @@ -31,20 +32,35 @@ type Build struct { LocalDeps bool `yaml:"local-deps"` } -// UpstreamSrcSignature specifies detached signature file for tarball -// and specifies the public key to be used to verify the signature. -type UpstreamSrcSignature struct { - DetachedSig string `yaml:"detached-sig"` - PubKey string `yaml:"public-key"` - SkipCheck bool `yaml:"skip-check"` +// DetachedSignature spec +// Specify either full URL of detached signature or how to derive it from source URL +type DetachedSignature struct { + FullURL string `yaml:"full-url"` + PubKey string `yaml:"public-key"` + OnUncompressed bool `yaml:"on-uncompressed"` +} + +// Signature spec +// Signature params to verify tarballs +type Signature struct { + SkipCheck bool `yaml:"skip-check"` + DetachedSignature DetachedSignature `yaml:"detached-sig"` +} + +// SourceBundle spec +// Used to generate the source url for srpm +type SourceBundle struct { + Name string `yaml:"name"` + SrcRepoParamsOverride srcconfig.SrcRepoParamsOverride `yaml:"override"` } // UpstreamSrc spec // Lists each source bundle(tarball/srpm) and // detached signature file for tarball. type UpstreamSrc struct { - Source string `yaml:"source"` - Signature UpstreamSrcSignature `yaml:"signature"` + SourceBundle SourceBundle `yaml:"source-bundle"` + FullURL string `yaml:"full-url"` + Signature Signature `yaml:"signature"` } // Package spec @@ -84,6 +100,27 @@ func (m Manifest) sanityCheck() error { return fmt.Errorf("No repo-bundle specified for Build in package %s", pkgSpec.Name) } + + for _, upStreamSrc := range pkgSpec.UpstreamSrc { + specifiedFullSrcURL := (upStreamSrc.FullURL != "") + specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) + if !specifiedFullSrcURL && !specifiedSrcBundle { + return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } + + if specifiedFullSrcURL && specifiedSrcBundle { + return fmt.Errorf( + "Conflicting sources for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } + + specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" + if specifiedFullSigURL && specifiedSrcBundle { + return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", + pkgSpec.Name) + } + } } return nil } diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 4cebf56..45bb33c 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -4,10 +4,11 @@ package manifest import ( - "github.com/stretchr/testify/require" "os" "testing" + "github.com/stretchr/testify/require" + "github.com/spf13/viper" "code.arista.io/eos/tools/eext/testutil" @@ -37,3 +38,43 @@ func TestManifest(t *testing.T) { testLoad(t, "pkg1") t.Log("Load test passed") } + +type manifestTestVariant struct { + TestPkg string + ManifestFile string + ExpectedErr string +} + +func TestManifestNegative(t *testing.T) { + t.Log("Create temporary working directory") + dir, err := os.MkdirTemp("", "manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + viper.Set("SrcDir", dir) + defer viper.Reset() + + testCases := map[string]manifestTestVariant{ + "testBundleAndFullURL": manifestTestVariant{ + TestPkg: "pkg2", + ManifestFile: "sampleManifest2.yaml", + ExpectedErr: "Conflicting sources for Build in package libpcap, provide either full-url or source-bundle", + }, + "testBundleAndSignature": manifestTestVariant{ + TestPkg: "pkg3", + ManifestFile: "sampleManifest3.yaml", + ExpectedErr: "Conflicting signatures for Build in package tcpdump, provide full-url or source-bundle", + }, + } + for testName, variant := range testCases { + t.Logf("%s: Copy sample manifest to test directory", testName) + testutil.SetupManifest(t, dir, variant.TestPkg, variant.ManifestFile) + + t.Logf("%s: Testing Load", testName) + _, err := LoadManifest(variant.TestPkg) + require.ErrorContains(t, err, variant.ExpectedErr) + t.Logf("%s: Load test passed", testName) + } +} diff --git a/manifest/testData/sampleManifest1.yaml b/manifest/testData/sampleManifest1.yaml index 9e02244..06868e2 100644 --- a/manifest/testData/sampleManifest1.yaml +++ b/manifest/testData/sampleManifest1.yaml @@ -3,7 +3,7 @@ package: - name: libpcap subdir: true upstream-sources: - - source: http://foo/libpcap.src.rpm + - full-url: http://foo/libpcap.src.rpm signature: skip-check: true type: srpm @@ -18,10 +18,42 @@ package: - name: bar - name: tcpdump upstream-sources: - - source: http://foo/tcpdump.tar.xz + - full-url: http://foo/tcpdump.tar.xz signature: - detached-sig: http://foo/tcpdump.tar.xz.sig - public-key: mrtparse/mrtparsePubKey.pem + detached-sig: + full-url: http://foo/tcpdump.tar.xz.sig + public-key: mrtparse/mrtparsePubKey.pem + type: tarball + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + local-deps: true + - name: binutils + upstream-sources: + - source-bundle: + name: srpm + override: + version: 1.1 + type: srpm + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + - name: libutils + upstream-sources: + - source-bundle: + name: tarball + override: + version: 1.2.1 + sig-suffix: .asc + signature: + skip-check: true + detached-sig: + public-key: mrtparse/mrtparsePubKey.pem + on-uncompressed: true type: tarball build: repo-bundle: diff --git a/manifest/testData/sampleManifest2.yaml b/manifest/testData/sampleManifest2.yaml new file mode 100644 index 0000000..cf0f7e4 --- /dev/null +++ b/manifest/testData/sampleManifest2.yaml @@ -0,0 +1,20 @@ +--- +package: + - name: libpcap + subdir: true + upstream-sources: + - full-url: http://foo/libpcap.src.rpm + source-bundle: + name: srpm + override: + version: 1.1.0 + type: srpm + build: + repo-bundle: + - name: foo + version: v1 + override: + rfoo: + enabled: true + exclude: "rfoo.rpm" + - name: bar diff --git a/manifest/testData/sampleManifest3.yaml b/manifest/testData/sampleManifest3.yaml new file mode 100644 index 0000000..8dab3dd --- /dev/null +++ b/manifest/testData/sampleManifest3.yaml @@ -0,0 +1,19 @@ +--- +package: + - name: tcpdump + upstream-sources: + - source-bundle: + name: tarball + override: + sig-suffix: .sig + signature: + detached-sig: + full-url: http://foo/tcpdump.tar.xz.sig + public-key: mrtparse/mrtparsePubKey.pem + type: tarball + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + local-deps: true diff --git a/srcconfig/srcconfig.go b/srcconfig/srcconfig.go new file mode 100644 index 0000000..f63d3b1 --- /dev/null +++ b/srcconfig/srcconfig.go @@ -0,0 +1,248 @@ +// Copyright (c) 2023 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package srcconfig + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "text/template" + + "code.arista.io/eos/tools/eext/util" + "github.com/spf13/viper" + "gopkg.in/yaml.v2" +) + +// SrcRepoParamsOverride spec +// Override default specs for source repo bundle in package manifest +type SrcRepoParamsOverride struct { + VersionOverride string `yaml:"version"` + SrcSuffixOverride string `yaml:"src-suffix"` + SigSuffixOverride string `yaml:"sig-suffix"` +} + +// SrcBundle spec +// To generate the source url for srpm +type SrcBundle struct { + URLFormat string `yaml:"url-format"` + DefaultSrcSuffix string `yaml:"default-src-suffix"` + DefaultSigSuffix string `yaml:"default-sig-suffix"` + VersionLabels map[string]string `yaml:"version-labels"` + HasDetachedSig bool `yaml:"has-detached-sig"` + urlFormatTemplate *template.Template +} + +// SrcConfig spec +// Mapping for all source repo bundles +type SrcConfig struct { + SrcBundle map[string]*SrcBundle `yaml:"source-bundle"` +} + +// SrcParams spec +// Updated source and signature url paths +type SrcParams struct { + SrcURL string + SignatureURL string +} + +func (s *SrcBundle) getTranslatedVersion(versionOverride string, errPrefix util.ErrPrefix) (string, error) { + var version string + if versionOverride == "" { + version = "default" + } else { + version = versionOverride + } + translatedVersion, isVersionLabel := s.VersionLabels[version] + if !isVersionLabel { + translatedVersion = version + } + + if translatedVersion == "default" { + return "", fmt.Errorf("%s No defaults specified for source-bundle, please specify a version override", + errPrefix) + } + + return translatedVersion, nil +} + +func (s *SrcBundle) getFormattedSigURL(srcFormattedURL, sigSuffixOverride string, onUncompressed bool) string { + var sigFormattedURL string + if s.HasDetachedSig { + var sigSuffix string + if sigSuffixOverride == "" { + sigSuffix = s.DefaultSigSuffix + } else { + sigSuffix = sigSuffixOverride + } + + var sigURLWithoutSuffix string + if onUncompressed { + sigURLWithoutSuffix = strings.TrimSuffix(srcFormattedURL, filepath.Ext(srcFormattedURL)) + } else { + sigURLWithoutSuffix = srcFormattedURL + } + sigFormattedURL = sigURLWithoutSuffix + sigSuffix + } else { + sigFormattedURL = "" + } + return sigFormattedURL +} + +func (s *SrcBundle) getSrcRepoParams( + pkgName string, + srcOverrideParams SrcRepoParamsOverride, + onUncompressed bool, + errPrefix util.ErrPrefix) ( + *SrcParams, error) { + + translatedVersion, err := s.getTranslatedVersion(srcOverrideParams.VersionOverride, errPrefix) + if err != nil { + return nil, err + } + + var srcSuffix string + if srcOverrideParams.SrcSuffixOverride == "" { + srcSuffix = s.DefaultSrcSuffix + } else { + srcSuffix = srcOverrideParams.SrcSuffixOverride + } + urlData := struct { + Host string + PathPrefix string + PkgName string + Version string + Suffix string + }{ + Host: viper.GetString("SrcRepoHost"), + PathPrefix: viper.GetString("SrcRepoPathPrefix"), + PkgName: pkgName, + Version: translatedVersion, + Suffix: srcSuffix, + } + + var urlBuf bytes.Buffer + if err := s.urlFormatTemplate.Execute(&urlBuf, urlData); err != nil { + return nil, fmt.Errorf("%sError executing template %s with data %v", + errPrefix, s.URLFormat, urlData) + } + srcFormattedURL := urlBuf.String() + + sigFormattedURL := s.getFormattedSigURL(srcFormattedURL, srcOverrideParams.SigSuffixOverride, onUncompressed) + + return &SrcParams{ + SrcURL: srcFormattedURL, + SignatureURL: sigFormattedURL, + }, nil +} + +func formatURLTemplate(urlFormat string, errPrefix util.ErrPrefix) (string, error) { + var urlBuf bytes.Buffer + urlData := struct { + Host string + PathPrefix string + }{ + Host: viper.GetString("SrcRepoHost"), + PathPrefix: viper.GetString("SrcRepoPathPrefix"), + } + + t, err := template.New("urlTemplate").Parse(urlFormat) + if err != nil { + return "", fmt.Errorf("%sError parsing source URL %s", errPrefix, urlFormat) + } + + if err := t.Execute(&urlBuf, urlData); err != nil { + return "", fmt.Errorf("%sError executing template %s with data %v", + errPrefix, urlFormat, urlData) + } + + return urlBuf.String(), nil +} + +func getSrcParamsWithoutBundle(srcFullURL, sigFullURL string, errPrefix util.ErrPrefix) (*SrcParams, error) { + srcURL, err := formatURLTemplate(srcFullURL, errPrefix) + if err != nil { + return nil, fmt.Errorf("%sUnable to format provided source url %s", errPrefix, srcFullURL) + } + sigURL, err := formatURLTemplate(sigFullURL, errPrefix) + if err != nil { + return nil, fmt.Errorf("%sUnable to format provided signature url %s", errPrefix, sigFullURL) + } + return &SrcParams{ + SrcURL: srcURL, + SignatureURL: sigURL, + }, nil +} + +func GetSrcParams( + pkgName string, + srcFullURL string, + bundleName string, + sigFullURL string, + srcOverrideParams SrcRepoParamsOverride, + onUncompressed bool, + srcConfig *SrcConfig, + errPrefix util.ErrPrefix) ( + *SrcParams, error) { + + var srcParams *SrcParams + var srcParamsErr error + if bundleName != "" { + reqSrcBundle, ok := srcConfig.SrcBundle[bundleName] + if !ok { + return nil, fmt.Errorf("%sSource bundle %s not found in manifest", errPrefix, bundleName) + } + srcParams, srcParamsErr = reqSrcBundle.getSrcRepoParams( + pkgName, + srcOverrideParams, + onUncompressed, + errPrefix) + } else { + srcParams, srcParamsErr = getSrcParamsWithoutBundle(srcFullURL, sigFullURL, errPrefix) + } + + if srcParamsErr != nil { + return nil, srcParamsErr + } + return srcParams, nil +} + +func LoadSrcConfig() (*SrcConfig, error) { + cfgPath := viper.GetString("SrcConfigFile") + _, statErr := os.Stat(cfgPath) + if statErr != nil { + if os.IsNotExist(statErr) { + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: %s doesn't exist", + cfgPath) + } + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: os.Stat on %s returned %s", + cfgPath, statErr) + } + + yamlContents, readErr := ioutil.ReadFile(cfgPath) + if readErr != nil { + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: ioutil.ReadFile on %s returned %s", + cfgPath, readErr) + } + + var config SrcConfig + if parseErr := yaml.UnmarshalStrict(yamlContents, &config); parseErr != nil { + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: Error parsing yaml file %s: %s", + cfgPath, parseErr) + } + + for bundleName, srcBundle := range config.SrcBundle { + templateName := "srcRepoBundle_" + bundleName + t, parseErr := template.New(templateName).Parse(srcBundle.URLFormat) + if parseErr != nil { + return nil, fmt.Errorf( + "srcconfig.LoadSrcConfig: Error parsing source %s for src repo-bundle %s", + srcBundle.URLFormat, bundleName) + } + srcBundle.urlFormatTemplate = t + } + return &config, nil +} diff --git a/srcconfig/srcconfig_test.go b/srcconfig/srcconfig_test.go new file mode 100644 index 0000000..7b6b019 --- /dev/null +++ b/srcconfig/srcconfig_test.go @@ -0,0 +1,150 @@ +// Copyright (c) 2022 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package srcconfig + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestRepoConfigSrpm(t *testing.T) { + viper.Set("SrcConfigFile", "testData/sample-srcconfig1.yaml") + viper.Set("SrcRepoHost", "http://foo.org") + viper.Set("SrcRepoPathPrefix", "foo") + defer viper.Reset() + + t.Log("Testing Load") + srcConfig, loadErr := LoadSrcConfig() + require.NoError(t, loadErr) + require.NotNil(t, srcConfig) + require.Contains(t, srcConfig.SrcBundle, "bundle-srpm") + srpmConfig := srcConfig.SrcBundle["bundle-srpm"] + require.NotNil(t, srpmConfig) + require.NotNil(t, srpmConfig.urlFormatTemplate) + t.Log("Load test PASSED") + + t.Log("Testing URL Format") + expectedURL := "http://foo.org/foo/pkg/pkg-1.0.src.rpm" + versions := []string{"1.0"} + for _, version := range versions { + srpmParams, paramsErr := GetSrcParams( + "pkg", // Package Name + "", // Full Source URL + "bundle-srpm", // Bundle Name + "", // Signature URL + SrcRepoParamsOverride{ + VersionOverride: version, + SrcSuffixOverride: "", + SigSuffixOverride: "", + }, + false, // onUncompressed + srcConfig, // Source Config + "", // Error Prefix + ) + require.NoError(t, paramsErr) + require.NotNil(t, srpmParams) + require.Equal(t, expectedURL, srpmParams.SrcURL) + } + t.Log("URL template test PASSED") +} + +type srcconfigTestVariant struct { + BundleName string + SrcOverrideParams SrcRepoParamsOverride + ExpectedSrcURL string + ExpectedSigURL string + OnUncompressed bool +} + +func TestRepoConfigTarBall(t *testing.T) { + viper.Set("SrcConfigFile", "testData/sample-srcconfig2.yaml") + viper.Set("SrcRepoHost", "http://foo.org") + viper.Set("SrcRepoPathPrefix", "foo") + defer viper.Reset() + + testCases := map[string]srcconfigTestVariant{ + "defaultSignatureTest": srcconfigTestVariant{ + BundleName: "bundle-tarball1", + SrcOverrideParams: SrcRepoParamsOverride{ + VersionOverride: "1.1", + SrcSuffixOverride: "", + SigSuffixOverride: "", + }, + ExpectedSrcURL: "http://foo.org/foo/tarballs/pkg/1.1/pkg.tar.gz", + ExpectedSigURL: "http://foo.org/foo/tarballs/pkg/1.1/pkg.tar.gz.sig", + OnUncompressed: false, + }, + "overridenSignatureTest": srcconfigTestVariant{ + BundleName: "bundle-tarball2", + SrcOverrideParams: SrcRepoParamsOverride{ + VersionOverride: "2.2", + SrcSuffixOverride: ".tar.bz2", + SigSuffixOverride: ".asc", + }, + ExpectedSrcURL: "http://foo.org/foo/tarballs/pkg/2.2/pkg.tar.bz2", + ExpectedSigURL: "http://foo.org/foo/tarballs/pkg/2.2/pkg.tar.asc", + OnUncompressed: true, + }, + } + + t.Log("Testing SrcConfig Load") + srcConfig, loadErr := LoadSrcConfig() + require.NoError(t, loadErr) + require.NotNil(t, srcConfig) + t.Log("SrcConfig Load test PASSED") + + for testName, variant := range testCases { + t.Logf("%s: Check bundle present", testName) + require.Contains(t, srcConfig.SrcBundle, variant.BundleName) + bundle := srcConfig.SrcBundle[variant.BundleName] + require.NotNil(t, bundle) + require.NotNil(t, bundle.urlFormatTemplate) + t.Logf("%s: Bundle test PASSED", testName) + + srcParams, paramsErr := GetSrcParams( + "pkg", // Package Name + "", // Source Full URL + variant.BundleName, // Bundle Name + "", // Signature Full URL + variant.SrcOverrideParams, // Override params + variant.OnUncompressed, // OnUncompressed + srcConfig, // Source Config + "", // Error Prefix + ) + + t.Logf("%s: URL template", testName) + require.NoError(t, paramsErr) + require.NotNil(t, srcParams) + require.Equal(t, variant.ExpectedSrcURL, srcParams.SrcURL) + t.Logf("%s: URL template PASSED", testName) + + t.Logf("%s: Signature path", testName) + require.Equal(t, variant.ExpectedSigURL, srcParams.SignatureURL) + t.Logf("%s: Signature path PASSED", testName) + } + + t.Log("Version assert Check") + versionTestName := "emptyVersionTest" + versionTestVariant := srcconfigTestVariant{ + BundleName: "bundle-tarball3", + SrcOverrideParams: SrcRepoParamsOverride{}, + ExpectedSrcURL: "", + ExpectedSigURL: "", + OnUncompressed: false, + } + versionTestExpError := "No defaults specified for source-bundle, please specify a version override" + _, paramsErr := GetSrcParams( + "pkg", // Package Name + "", // Full Source URL + versionTestVariant.BundleName, // Bundle Name + "", // Signature Full URL + versionTestVariant.SrcOverrideParams, // Override Params + versionTestVariant.OnUncompressed, // OnUncompressed + srcConfig, // Source Config + "") // Error Prefix + require.ErrorContains(t, paramsErr, versionTestExpError) + t.Logf("%s: Version check PASSED", versionTestName) +} diff --git a/srcconfig/testData/sample-srcconfig1.yaml b/srcconfig/testData/sample-srcconfig1.yaml new file mode 100644 index 0000000..c895be9 --- /dev/null +++ b/srcconfig/testData/sample-srcconfig1.yaml @@ -0,0 +1,5 @@ +--- +# yamllint disable rule:line-length +source-bundle: + bundle-srpm: + url-format: "{{.Host}}/{{.PathPrefix}}/{{.PkgName}}/{{.PkgName}}-{{.Version}}.src.rpm" diff --git a/srcconfig/testData/sample-srcconfig2.yaml b/srcconfig/testData/sample-srcconfig2.yaml new file mode 100644 index 0000000..02bc5ea --- /dev/null +++ b/srcconfig/testData/sample-srcconfig2.yaml @@ -0,0 +1,20 @@ +--- +# yamllint disable rule:line-length +source-bundle: + bundle-tarball1: + url-format: "{{.Host}}/{{.PathPrefix}}/tarballs/{{.PkgName}}/{{.Version}}/{{.PkgName}}{{.Suffix}}" + default-src-suffix: .tar.gz + default-sig-suffix: .sig + has-detached-sig: true + + bundle-tarball2: + url-format: "{{.Host}}/{{.PathPrefix}}/tarballs/{{.PkgName}}/{{.Version}}/{{.PkgName}}{{.Suffix}}" + default-src-suffix: .tar.bz2 + default-sig-suffix: .sig + has-detached-sig: true + + bundle-tarball3: + url-format: "{{.Host}}/{{.PathPrefix}}/tarballs/{{.PkgName}}/{{.Version}}/{{.PkgName}}{{.Suffix}}" + default-src-suffix: .tar.gz + default-sig-suffix: .asc + has-detached-sig: true diff --git a/testutil/testutil.go b/testutil/testutil.go index 683af10..1cf3444 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -109,6 +109,9 @@ func SetupViperConfig( depsDir string, repoHost string, dnfConfigFile string, + srcRepoHost string, + srcConfigFile string, + srcRepoPathPrefix string, ) { if srcDir == "" { viper.Set("SrcDir", "testData") @@ -130,6 +133,24 @@ func SetupViperConfig( } else { viper.Set("DnfConfigFile", dnfConfigFile) } + if srcRepoHost == "" { + viper.Set("SrcRepoHost", + "https://artifactory.infra.corp.arista.io") + } else { + viper.Set("SrcRepoHost", srcRepoHost) + } + if srcConfigFile == "" { + viper.Set("SrcConfigFile", + "../configfiles/srcconfig.yaml") + } else { + viper.Set("SrcConfigFile", srcConfigFile) + } + if srcRepoPathPrefix == "" { + viper.Set("SrcRepoPathPrefix", + "artifactory/eext-sources") + } else { + viper.Set("SrcRepoPathPrefix", srcRepoPathPrefix) + } viper.Set("MockCfgTemplate", "../configfiles/mock.cfg.template") viper.Set("PkiPath",