eext: Adding source bundle capability for eext packages to invoke using repoName

cmd/common.go

@@ -42,6 +42,12 @@ func SetViperDefaults() {
 		"http://artifactory.infra.corp.arista.io")
 	viper.SetDefault("DnfConfigFile",
 		"/usr/share/eext/dnfconfig.yaml")
+	viper.SetDefault("SrcRepoHost",
+		"http://artifactory.infra.corp.arista.io")
+	viper.SetDefault("SrcConfigFile",
+		"/usr/share/eext/srcconfig.yaml")
+	viper.SetDefault("SrcRepoPathPrefix",
+		"artifactory/eext-sources")
 	viper.SetDefault("PkiPath",
 		"/etc/pki/eext")
 	viper.SetDefault("SrcEnvPrefix",

cmd/create_srpm_test.go

@@ -35,11 +35,15 @@ func testCreateSrpm(t *testing.T,
 
 	t.Logf("DestDir: %s", destDir)
 	testutil.SetupViperConfig(
-		"", // srcDir
-		workingDir, destDir,
-		"", // depsDir
-		"", // repoHost,
-		"", // dnfConfigFile
+		"",                                    // srcDir
+		workingDir,                            // workingDir
+		destDir,                               // destDir
+		"",                                    // depsDir
+		"",                                    // repoHost,
+		"",                                    // dnfConfigFile
+		"",                                    // srcRepoHost
+		"testData/configfiles/srcconfig.yaml", // srcConfigFile
+		"",                                    // srcRepoPathPrefix
 	)
 	defer viper.Reset()
 	if sources != nil {

cmd/mock_test.go

@@ -69,6 +69,9 @@ func testMock(t *testing.T, setupSrcEnv bool) {
 		"", // depsDir
 		"", // repoHost,
 		"", // dnfConfigFile
+		"", // srcRepoHost
+		"", // srcConfigFile
+		"", // srcRepoPathPrefix
 	)
 
 	args := []string{"create-srpm", "--repo", repoName}

cmd/testData/configfiles/srcconfig.yaml

@@ -0,0 +1,5 @@
+---
+# yamllint disable rule:line-length
+source-bundle:
+  srpm:
+    url-format: "{{.Host}}/{{.PathPrefix}}/eext-testData/{{.PkgName}}-{{.Version}}.src.rpm"

cmd/testData/debugedit-1/eext.yaml

@@ -3,7 +3,7 @@
 package:
   - name: debugedit
     upstream-sources:
-      - source: https://artifactory.infra.corp.arista.io/artifactory/eext-sources/eext-testData/debugedit-5.0-3.el9.src.rpm
+      - full-url: "{{.Host}}/{{.PathPrefix}}/eext-testData/debugedit-5.0-3.el9.src.rpm"
     type: srpm
     build:
       repo-bundle:

cmd/testData/debugedit-1/spec/debugedit.spec

@@ -4,7 +4,7 @@
 
 Name: debugedit
 Version: 5.0
-Release: %{?release:%{release}}%{!?release:eng}
+Release: %{?eext_release:%{eext_release}}%{!?eext_release:eng}
 Summary: Tools for debuginfo creation
 License: GPLv3+ and GPLv2+ and LGPLv2+
 URL: https://sourceware.org/debugedit/

cmd/testData/debugedit-2/eext.yaml

@@ -3,7 +3,10 @@
 package:
   - name: debugedit
     upstream-sources:
-      - source: https://artifactory.infra.corp.arista.io/artifactory/eext-sources/eext-testData/debugedit-5.0-3.el9.src.rpm
+      - source-bundle:
+          name: srpm
+          override:
+            version: 5.0-3.el9
     type: unmodified-srpm
     build:
       repo-bundle:

cmd/testData/mrtparse-1/eext.yaml

@@ -3,10 +3,11 @@
 package:
   - name: mrtparse
     upstream-sources:
-      - source: file:///mrtparse-2.0.1.tar.gz
+      - full-url: file:///mrtparse-2.0.1.tar.gz
         signature:
-          detached-sig: file:///mrtparse-2.0.1.tar.gz.sig
-          public-key: mrtparse/mrtparsePubKey.pem
+          detached-sig:
+            full-url: file:///mrtparse-2.0.1.tar.gz.sig
+            public-key: mrtparse/mrtparsePubKey.pem
     type: tarball
     build:
       include:

cmd/testData/mrtparse-1/spec/mrtparse.spec

@@ -5,17 +5,17 @@ Patch1000: arista-remove-default-sigpipe.patch
 Summary:        Tool for parsing routing information dump files in MRT format
 Name:           mrtparse
 Version:        2.0.1
-Release:        %{?release:%{release}}%{!?release:eng}
+Release:        %{?eext_release:%{eext_release}}%{!?eext_release:eng}
 
-License:        Apache 
+License:        Apache
 Source0:        mrtparse-2.0.1.tar.gz
 Url:            https://github.com/YoshiyukiYamauchi/mrtparse
 BuildArch:      noarch
 
 %description
-mrtprse is a module to read and analyze the MRT format data. The MRT format data 
-can be used to export routing protocol messages, state changes, and routing 
-information base contents, and is standardized in RFC6396. Programs like Quagga 
+mrtprse is a module to read and analyze the MRT format data. The MRT format data
+can be used to export routing protocol messages, state changes, and routing
+information base contents, and is standardized in RFC6396. Programs like Quagga
 / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data.
 
 %package -n python3-mrtparse
@@ -25,9 +25,9 @@ BuildRequires:  python3-setuptools
 %{?python_provide:%python_provide python3-mrtparse}
 
 %description -n python3-mrtparse
-mrtprse is a module to read and analyze the MRT format data. The MRT format data 
-can be used to export routing protocol messages, state changes, and routing 
-information base contents, and is standardized in RFC6396. Programs like Quagga 
+mrtprse is a module to read and analyze the MRT format data. The MRT format data
+can be used to export routing protocol messages, state changes, and routing
+information base contents, and is standardized in RFC6396. Programs like Quagga
 / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data.
 
 %prep

cmd/testData/mrtparse-2/eext.yaml

@@ -4,7 +4,7 @@ package:
   - name: mrtparse
     subdir: true
     upstream-sources:
-      - source: file:///mrtparse-2.0.1.tar.gz
+      - full-url: file:///mrtparse-2.0.1.tar.gz
         signature:
           skip-check: true
     type: tarball

cmd/testData/mrtparse-2/mrtparse/spec/mrtparse.spec

@@ -5,17 +5,17 @@ Patch1000: arista-remove-default-sigpipe.patch
 Summary:        Tool for parsing routing information dump files in MRT format
 Name:           mrtparse
 Version:        2.0.1
-Release:        %{?release:%{release}}%{!?release:eng}
+Release:        %{?eext_release:%{eext_release}}%{!?eext_release:eng}
 
-License:        Apache 
+License:        Apache
 Source0:        mrtparse-2.0.1.tar.gz
 Url:            https://github.com/YoshiyukiYamauchi/mrtparse
 BuildArch:      noarch
 
 %description
-mrtprse is a module to read and analyze the MRT format data. The MRT format data 
-can be used to export routing protocol messages, state changes, and routing 
-information base contents, and is standardized in RFC6396. Programs like Quagga 
+mrtprse is a module to read and analyze the MRT format data. The MRT format data
+can be used to export routing protocol messages, state changes, and routing
+information base contents, and is standardized in RFC6396. Programs like Quagga
 / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data.
 
 %package -n python3-mrtparse
@@ -25,9 +25,9 @@ BuildRequires:  python3-setuptools
 %{?python_provide:%python_provide python3-mrtparse}
 
 %description -n python3-mrtparse
-mrtprse is a module to read and analyze the MRT format data. The MRT format data 
-can be used to export routing protocol messages, state changes, and routing 
-information base contents, and is standardized in RFC6396. Programs like Quagga 
+mrtprse is a module to read and analyze the MRT format data. The MRT format data
+can be used to export routing protocol messages, state changes, and routing
+information base contents, and is standardized in RFC6396. Programs like Quagga
 / Zebra, BIRD, OpenBGPD and PyRT can dump the MRT fotmat data.
 
 %prep

configfiles/srcconfig.yaml

@@ -0,0 +1,10 @@
+---
+# yamllint disable rule:line-length
+source-bundle:
+  srpm:
+    url-format: "{{.Host}}/{{.PathPrefix}}/source-bundles/srpm/{{.PkgName}}/{{.PkgName}}-{{.Version}}.src.rpm"
+  tarball:
+    url-format: "{{.Host}}/{{.PathPrefix}}/source-bundles/tarball/{{.PkgName}}/{{.Version}}/{{.PkgName}}-{{.Version}}{{.Suffix}}"
+    default-src-suffix: .tar.gz
+    default-sig-suffix: .sig
+    has-detached-sig: true

impl/checkenv.go

@@ -21,6 +21,7 @@ func CheckEnv() error {
 	destDir := viper.GetString("DestDir")
 	mockCfgTemplate := viper.GetString("MockCfgTemplate")
 	dnfConfigFile := viper.GetString("DnfConfigFile")
+	srcConfigFile := viper.GetString("SrcConfigFile")
 	pkiPath := viper.GetString("PkiPath")
 
 	var aggError string
@@ -65,6 +66,11 @@ func CheckEnv() error {
 		failed = true
 	}
 
+	if err := util.CheckPath(srcConfigFile, false, false); err != nil {
+		aggError += fmt.Sprintf("\ntrouble with SrcConfigFile: %s", err)
+		failed = true
+	}
+
 	if err := util.CheckPath(pkiPath, true, false); err != nil {
 		aggError += fmt.Sprintf("\ntrouble with PkiPath: %s", err)
 		failed = true

impl/create_srpm.go

@@ -13,6 +13,7 @@ import (
 	"golang.org/x/exp/slices"
 
 	"code.arista.io/eos/tools/eext/manifest"
+	"code.arista.io/eos/tools/eext/srcconfig"
 	"code.arista.io/eos/tools/eext/util"
 )
 
@@ -30,6 +31,7 @@ type srpmBuilder struct {
 	errPrefixBase util.ErrPrefix
 	errPrefix     util.ErrPrefix
 	upstreamSrc   []upstreamSrcSpec
+	srcConfig     *srcconfig.SrcConfig
 }
 
 // CreateSrpmExtraCmdlineArgs is a bundle of extra args for impl.CreateSrpm
@@ -86,42 +88,51 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 	}
 
 	for _, upstreamSrcFromManifest := range bldr.pkgSpec.UpstreamSrc {
-		if upstreamSrcFromManifest.Source == "" {
-			return fmt.Errorf("%ssource not specified for upstream-sources entry",
-				bldr.errPrefix)
+		srcParams, err := srcconfig.GetSrcParams(
+			bldr.pkgSpec.Name,
+			upstreamSrcFromManifest.FullURL,
+			upstreamSrcFromManifest.SourceBundle.Name,
+			upstreamSrcFromManifest.Signature.DetachedSignature.FullURL,
+			upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride,
+			upstreamSrcFromManifest.Signature.DetachedSignature.OnUncompressed,
+			bldr.srcConfig,
+			bldr.errPrefix)
+		if err != nil {
+			return fmt.Errorf("%sUnable to get source params for %s",
+				err, upstreamSrcFromManifest.SourceBundle.Name)
 		}
 
 		var downloadErr error
 		upstreamSrc := upstreamSrcSpec{}
 
-		bldr.log("downloading %s", upstreamSrcFromManifest.Source)
+		bldr.log("downloading %s", srcParams.SrcURL)
 		// Download source
 		if upstreamSrc.sourceFile, downloadErr = download(
-			upstreamSrcFromManifest.Source,
+			srcParams.SrcURL,
 			downloadDir,
 			repo, pkg, isPkgSubdirInRepo,
 			bldr.errPrefix); downloadErr != nil {
 			return downloadErr
 		}
 		bldr.log("downloaded")
 
-		signatureSpec := upstreamSrcFromManifest.Signature
-		upstreamSrc.skipSigCheck = signatureSpec.SkipCheck
+		upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck
+		pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey
 
-		if bldr.pkgSpec.Type == "tarball" && !signatureSpec.SkipCheck {
-			if signatureSpec.DetachedSig == "" || signatureSpec.PubKey == "" {
+		if bldr.pkgSpec.Type == "tarball" && !upstreamSrc.skipSigCheck {
+			if srcParams.SignatureURL == "" || pubKey == "" {
 				return fmt.Errorf("%sNo detached-signature/public-key specified for upstream-sources entry %s",
-					bldr.errPrefix, upstreamSrcFromManifest.Source)
+					bldr.errPrefix, srcParams.SrcURL)
 			}
 			if upstreamSrc.sigFile, downloadErr = download(
-				signatureSpec.DetachedSig,
+				srcParams.SignatureURL,
 				downloadDir,
 				repo, pkg, isPkgSubdirInRepo,
 				bldr.errPrefix); downloadErr != nil {
 				return downloadErr
 			}
 
-			pubKeyPath := filepath.Join(getDetachedSigDir(), signatureSpec.PubKey)
+			pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey)
 			if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil {
 				return fmt.Errorf("%sCannot find public-key at path %s",
 					bldr.errPrefix, pubKeyPath)
@@ -130,11 +141,11 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 		} else if bldr.pkgSpec.Type == "srpm" || bldr.pkgSpec.Type == "unmodified-srpm" {
 			// We don't expect SRPMs to have detached signature or
 			// to be validated with a public-key specified in manifest.
-			if signatureSpec.DetachedSig != "" {
+			if srcParams.SignatureURL != "" {
 				return fmt.Errorf("%sUnexpected detached-sig specified for SRPM",
 					bldr.errPrefix)
 			}
-			if signatureSpec.PubKey != "" {
+			if pubKey != "" {
 				return fmt.Errorf("%sUnexpected public-key specified for SRPM",
 					bldr.errPrefix)
 			}
@@ -381,7 +392,7 @@ func (bldr *srpmBuilder) build(prep bool) error {
 
 	if rpmReleaseMacro != "" {
 		rpmbuildArgs = append(rpmbuildArgs, []string{
-			"--define", fmt.Sprintf("release %s", rpmReleaseMacro),
+			"--define", fmt.Sprintf("eext_release %s", rpmReleaseMacro),
 		}...)
 	}
 	rpmbuildArgs = append(rpmbuildArgs, specFile)
@@ -527,6 +538,11 @@ func CreateSrpm(repo string, pkg string, extraArgs CreateSrpmExtraCmdlineArgs) e
 		return loadManifestErr
 	}
 
+	srcConfig, err := srcconfig.LoadSrcConfig()
+	if err != nil {
+		return err
+	}
+
 	var pkgSpecified bool = (pkg != "")
 	found := !pkgSpecified
 	for _, pkgSpec := range repoManifest.Package {
@@ -541,6 +557,7 @@ func CreateSrpm(repo string, pkg string, extraArgs CreateSrpmExtraCmdlineArgs) e
 			repo:          repo,
 			skipBuildPrep: extraArgs.SkipBuildPrep,
 			errPrefixBase: errPrefixBase,
+			srcConfig:     srcConfig,
 		}
 		bldr.setupStageErrPrefix("")
 

impl/mock_cfg.go

@@ -58,7 +58,7 @@ func (cfgBldr *mockCfgBuilder) populateTemplateData() error {
 
 	cfgBldr.templateData.Macros = make(map[string]string)
 	if cfgBldr.rpmReleaseMacro != "" {
-		cfgBldr.templateData.Macros["release"] = cfgBldr.rpmReleaseMacro
+		cfgBldr.templateData.Macros["eext_release"] = cfgBldr.rpmReleaseMacro
 	}
 
 	if cfgBldr.eextSignature != "" {

impl/mock_cfg_test.go

@@ -45,7 +45,10 @@ func testMockConfig(t *testing.T, chained bool) {
 		destDir,
 		"",                        // depsDir
 		"https://foo.org",         // repoHost
-		"testData/dnfconfig.yaml", //dnfConfigFile
+		"testData/dnfconfig.yaml", // dnfConfigFile
+		"",                        // srcRepoHost
+		"",                        // srcConfigFile
+		"",                        // srcRepoPathPrefix
 	)
 	defer viper.Reset()
 

impl/setupDeps_test.go

@@ -49,6 +49,9 @@ func TestSetupDeps(t *testing.T) {
 		depsDir,
 		"", // repoHost
 		"", // dnf config file
+		"", // src repo host
+		"", // src config file
+		"", // src repo path prefix
 	)
 	defer viper.Reset()
 

impl/testData/expected-mock.cfg

@@ -18,7 +18,7 @@ config_opts['target_arch'] = "x86_64"
 
 # Autogenerated macros
 config_opts['macros']['distribution'] = 'eextsig=my-signature'
-config_opts['macros']['release'] = 'my-release'
+config_opts['macros']['eext_release'] = 'my-release'
 
 # Autogenerated dnf.conf
 config_opts['dnf.conf'] = """

impl/testData/manifest.yaml

@@ -2,7 +2,7 @@
 package:
   - name: pkg1
     upstream-sources:
-      - source: http://foo.org/pkg1.src.rpm
+      - full-url: http://foo.org/pkg1.src.rpm
     type: srpm
     build:
       repo-bundle: