Add support for git repo as upstream source

eext currently supports 'srpm' and 'tarball' as upstream sources for
packages. Some users wanted to use an upstream git repo directly as
source for security purposes. Hence we add support for eext to use an
upstream git repo as an upstream source.

The git upstream source requires 'url' and 'revision' to be specified.
'url': Web url to the upstream git repo
'revision': Commit hash or release tag of the git repo

Users can verify their upstream git repo at the revision provided, by
specifying the corresponding public key. Note that this verification
only works if the commit/tag are signed, since we use 'git-verify' to
validate. For unsigned commits/tags, user need to enable 'skip-check'
in the signature field (not recommended though, since this may introduce
security vulnerabilities).

Another rule users need to adhere to while using git repo as upstream,
is that the spec file in 'spec/' folder should be named '<pkgName>.spec'
where 'pkgName' is the same as mentioned in eext.yaml.
And within the spec file, please ensure that the 'Source0:' field points
to '<pkgName>-<version>.tar.gz', where 'pkgName' is the name mentioned
in eext.yaml, and 'version' is derived from the 'revision' provided as:
1. If the revision is a TAG, then use TAG as 'version'.
2. If the revision is a COMMIT,
    a. If SHORT COMMIT, use SHORT COMMIT as 'version'
    b. Else use the first 7 charachters of LONG COMMIT as 'version'

impl/common.go

@@ -175,6 +175,96 @@ func checkRepo(repo string, pkg string, isPkgSubdirInRepo bool,
 	return nil
 }
 
+func getRpmNameFromSpecFile(repo, pkg string, isPkgSubdirInRepo bool) (string, error) {
+	pkgDirInRepo := getPkgDirInRepo(repo, pkg, isPkgSubdirInRepo)
+	specFileName := "spec/" + pkg + ".spec"
+	specFilePath := filepath.Join(pkgDirInRepo, specFileName)
+	if err := util.CheckPath(specFilePath, false, false); err != nil {
+		return "", fmt.Errorf("%s spec file not found for %s", specFilePath, pkg)
+	}
+
+	cmd := []string{"-q", "--srpm", "--qf", "%{NAME}-%{VERSION}", specFilePath}
+	rpmName, err := util.CheckOutput("rpmspec", cmd...)
+	if err != nil {
+		return "", fmt.Errorf("cannot query spec file %s for %s", specFilePath, pkg)
+	}
+
+	return rpmName, nil
+}
+
+// We aren't using 'git clone' since it is slow for large repos.
+// This method is faster and only pulls necessary changes.
+func cloneGitRepo(pkg, srcURL, revision string) (string, error) {
+	// Cloning the git repo to a temporary directory
+	// We won't delete the tmpDir here, since we need it to verify the git repo.
+	cloneDir, err := os.MkdirTemp("", pkg)
+	if err != nil {
+		return "", fmt.Errorf("error while creating tempDir for %s, %s", pkg, err)
+	}
+	// Init the dir as a git repo
+	err = util.RunSystemCmdInDir(cloneDir, "git", "init")
+	if err != nil {
+		return "", fmt.Errorf("git init at %s failed: %s", cloneDir, err)
+	}
+	// Add the srcURL as the origin for the repo
+	err = util.RunSystemCmdInDir(cloneDir, "git", "remote", "add", "origin", srcURL)
+	if err != nil {
+		return "", fmt.Errorf("adding %s as git remote failed: %s", srcURL, err)
+	}
+	// Fetch repo tags, for user inputs revision as TAG
+	err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "--tags")
+	if err != nil {
+		return "", fmt.Errorf("fetching tags failed for %s: %s", pkg, err)
+	}
+	// Fetch the code changes for the provided revision
+	err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "origin", revision)
+	if err != nil {
+		return "", fmt.Errorf("fetching revision %s failed for %s: %s", revision, pkg, err)
+	}
+	// Pull code to repo at provided revision
+	err = util.RunSystemCmdInDir(cloneDir, "git", "reset", "--hard", "FETCH_HEAD")
+	if err != nil {
+		return "", fmt.Errorf("fetching HEAD at %s failed: %s", revision, err)
+	}
+
+	return cloneDir, nil
+}
+
+// Download the git repo, and create a tarball at the provided commit/tag.
+func archiveGitRepo(srcURL, targetDir, version, revision, repo, pkg string, isPkgSubdirInRepo bool,
+	errPrefix util.ErrPrefix) (string, string, error) {
+	cloneDir, err := cloneGitRepo(pkg, srcURL, revision)
+	if err != nil {
+		return "", "", fmt.Errorf("%s cloning git repo failed: %s", errPrefix, err)
+	}
+
+	// User should ensure the same fileName is specified in .spec file.
+	// Format to follow is "<pkgName>-<version>.tar.gz"
+	// where the version is derived from the revision provided in eext.yaml.
+	// 1. Tag
+	// 2. Short Commit Hash
+	// 3. First 7 charachters of Long Commit Hash
+	gitArchiveFile := pkg + "-" + version + ".tar.gz"
+	gitArchiveFilePath := filepath.Join(targetDir, gitArchiveFile)
+	prefix, err := getRpmNameFromSpecFile(repo, pkg, isPkgSubdirInRepo)
+	if err != nil {
+		return "", "", err
+	}
+
+	// Create the tarball from the specified commit/tag revision
+	archiveCmd := []string{"archive",
+		"--prefix", prefix + "/",
+		"-o", gitArchiveFilePath,
+		revision,
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", archiveCmd...)
+	if err != nil {
+		return "", "", fmt.Errorf("%sgit archive of %s failed: %s %v", errPrefix, pkg, err, archiveCmd)
+	}
+
+	return gitArchiveFile, cloneDir, nil
+}
+
 // Download the resource srcURL to targetDir
 // srcURL could be URL or file path
 // If it is a file:// path, root directory is the

impl/create_srpm.go

@@ -23,6 +23,7 @@ type upstreamSrcSpec struct {
 	sigFile      string
 	pubKeyPath   string
 	skipSigCheck bool
+	gitSpec      util.GitSpec
 }
 
 type srpmBuilder struct {
@@ -89,9 +90,13 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 	}
 
 	for _, upstreamSrcFromManifest := range bldr.pkgSpec.UpstreamSrc {
+		upstreamUrl := upstreamSrcFromManifest.FullURL
+		if bldr.pkgSpec.Type == "git" {
+			upstreamUrl = upstreamSrcFromManifest.GitSpec.Url
+		}
 		srcParams, err := srcconfig.GetSrcParams(
 			bldr.pkgSpec.Name,
-			upstreamSrcFromManifest.FullURL,
+			upstreamUrl,
 			upstreamSrcFromManifest.SourceBundle.Name,
 			upstreamSrcFromManifest.Signature.DetachedSignature.FullURL,
 			upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride,
@@ -106,16 +111,43 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 		var downloadErr error
 		upstreamSrc := upstreamSrcSpec{}
 
-		bldr.log("downloading %s", srcParams.SrcURL)
-		// Download source
-		if upstreamSrc.sourceFile, downloadErr = download(
-			srcParams.SrcURL,
-			downloadDir,
-			repo, pkg, isPkgSubdirInRepo,
-			bldr.errPrefix); downloadErr != nil {
-			return downloadErr
+		if bldr.pkgSpec.Type == "git" {
+			bldr.log("creating tarball for %s from repo %s", pkg, srcParams.SrcURL)
+			spec := util.GitSpec{
+				SrcUrl:   srcParams.SrcURL,
+				Revision: upstreamSrcFromManifest.GitSpec.Revision,
+			}
+			version, err := spec.GetVersionFromRevision()
+			if err != nil {
+				return fmt.Errorf("error while getting file version: %s", err)
+			}
+			revision := spec.Revision
+			var clonedDir string
+			upstreamSrc.sourceFile, clonedDir, downloadErr = archiveGitRepo(
+				srcParams.SrcURL,
+				downloadDir,
+				version, revision,
+				repo, pkg, isPkgSubdirInRepo,
+				bldr.errPrefix)
+			if downloadErr != nil {
+				return downloadErr
+			}
+
+			spec.ClonedDir = clonedDir
+			upstreamSrc.gitSpec = spec
+			bldr.log("tarball created")
+		} else {
+			bldr.log("downloading %s", srcParams.SrcURL)
+			// Download source
+			if upstreamSrc.sourceFile, downloadErr = download(
+				srcParams.SrcURL,
+				downloadDir,
+				repo, pkg, isPkgSubdirInRepo,
+				bldr.errPrefix); downloadErr != nil {
+				return downloadErr
+			}
+			bldr.log("downloaded")
 		}
-		bldr.log("downloaded")
 
 		upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck
 		pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey
@@ -150,6 +182,25 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 				return fmt.Errorf("%sUnexpected public-key specified for SRPM",
 					bldr.errPrefix)
 			}
+		} else if bldr.pkgSpec.Type == "git" {
+			// TODO: Should we enable skip-check by default?
+			// Skip signature check for git repo if 'signature' field is empty in eext.yaml
+			// specifiedSignature := (upstreamSrcFromManifest.Signature != manifest.Signature{})
+			// skipCheck := upstreamSrcFromManifest.Signature.SkipCheck
+			// upstreamSrc.skipSigCheck = !specifiedSignature || skipCheck
+
+			if !upstreamSrc.skipSigCheck {
+				if pubKey == "" {
+					return fmt.Errorf("%sexpected public-key for %s to verify git repo",
+						bldr.errPrefix, pkg)
+				}
+				pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey)
+				if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil {
+					return fmt.Errorf("%sCannot find public-key at path %s",
+						bldr.errPrefix, pubKeyPath)
+				}
+				upstreamSrc.pubKeyPath = pubKeyPath
+			}
 		}
 
 		bldr.upstreamSrc = append(bldr.upstreamSrc, upstreamSrc)
@@ -216,6 +267,17 @@ func (bldr *srpmBuilder) verifyUpstream() error {
 		if err := bldr.verifyUpstreamSrpm(); err != nil {
 			return err
 		}
+	} else if bldr.pkgSpec.Type == "git" {
+		for _, upstreamSrc := range bldr.upstreamSrc {
+			if !upstreamSrc.skipSigCheck {
+				err := util.VerifyGitSignature(upstreamSrc.pubKeyPath, upstreamSrc.gitSpec, bldr.errPrefix)
+				if err != nil {
+					return err
+				}
+			}
+			// Deleting cloned git repo since we no longer require it.
+			os.RemoveAll(upstreamSrc.gitSpec.ClonedDir)
+		}
 	} else {
 		downloadDir := getDownloadDir(bldr.pkgSpec.Name)
 		for _, upstreamSrc := range bldr.upstreamSrc {
@@ -271,7 +333,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeSrpm() error {
 // also checks tarball signature
 func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error {
 
-	supportedTypes := []string{"tarball", "standalone"}
+	supportedTypes := []string{"tarball", "standalone", "git"}
 	if !slices.Contains(supportedTypes, bldr.pkgSpec.Type) {
 		panic(fmt.Sprintf("%ssetupRpmbuildTreeNonSrpm called for unsupported type %s",
 			bldr.errPrefix, bldr.pkgSpec.Type))
@@ -284,7 +346,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error {
 		return err
 	}
 
-	if bldr.pkgSpec.Type == "tarball" {
+	if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "git" {
 		downloadDir := getDownloadDir(bldr.pkgSpec.Name)
 		for _, upstreamSrc := range bldr.upstreamSrc {
 			upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile)
@@ -379,7 +441,7 @@ func (bldr *srpmBuilder) setupRpmbuildTree() error {
 		if err := bldr.setupRpmbuildTreeSrpm(); err != nil {
 			return err
 		}
-	} else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" {
+	} else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" || bldr.pkgSpec.Type == "git" {
 		if err := bldr.setupRpmbuildTreeNonSrpm(); err != nil {
 			return err
 		}

impl/testData/upstream-git-repo-1/eext.yaml

@@ -0,0 +1,14 @@
+---
+package:
+  - name: libpcap
+    upstream-sources:
+      - git:
+          url: https://github.com/the-tcpdump-group/libpcap
+          revision: libpcap-1.10.1
+        signature:
+          detached-sig:
+            public-key: tcpdump/tcpdumpPubKey.pem
+    type: git
+    build:
+      repo-bundle:
+        - name: el9