diff --git a/impl/common.go b/impl/common.go index b6d391f..ab49348 100644 --- a/impl/common.go +++ b/impl/common.go @@ -175,6 +175,96 @@ func checkRepo(repo string, pkg string, isPkgSubdirInRepo bool, return nil } +func getRpmNameFromSpecFile(repo, pkg string, isPkgSubdirInRepo bool) (string, error) { + pkgDirInRepo := getPkgDirInRepo(repo, pkg, isPkgSubdirInRepo) + specFileName := "spec/" + pkg + ".spec" + specFilePath := filepath.Join(pkgDirInRepo, specFileName) + if err := util.CheckPath(specFilePath, false, false); err != nil { + return "", fmt.Errorf("%s spec file not found for %s", specFilePath, pkg) + } + + cmd := []string{"-q", "--srpm", "--qf", "%{NAME}-%{VERSION}", specFilePath} + rpmName, err := util.CheckOutput("rpmspec", cmd...) + if err != nil { + return "", fmt.Errorf("cannot query spec file %s for %s", specFilePath, pkg) + } + + return rpmName, nil +} + +// We aren't using 'git clone' since it is slow for large repos. +// This method is faster and only pulls necessary changes. +func cloneGitRepo(pkg, srcURL, revision string) (string, error) { + // Cloning the git repo to a temporary directory + // We won't delete the tmpDir here, since we need it to verify the git repo. + cloneDir, err := os.MkdirTemp("", pkg) + if err != nil { + return "", fmt.Errorf("error while creating tempDir for %s, %s", pkg, err) + } + // Init the dir as a git repo + err = util.RunSystemCmdInDir(cloneDir, "git", "init") + if err != nil { + return "", fmt.Errorf("git init at %s failed: %s", cloneDir, err) + } + // Add the srcURL as the origin for the repo + err = util.RunSystemCmdInDir(cloneDir, "git", "remote", "add", "origin", srcURL) + if err != nil { + return "", fmt.Errorf("adding %s as git remote failed: %s", srcURL, err) + } + // Fetch repo tags, for user inputs revision as TAG + err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "--tags") + if err != nil { + return "", fmt.Errorf("fetching tags failed for %s: %s", pkg, err) + } + // Fetch the code changes for the provided revision + err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "origin", revision) + if err != nil { + return "", fmt.Errorf("fetching revision %s failed for %s: %s", revision, pkg, err) + } + // Pull code to repo at provided revision + err = util.RunSystemCmdInDir(cloneDir, "git", "reset", "--hard", "FETCH_HEAD") + if err != nil { + return "", fmt.Errorf("fetching HEAD at %s failed: %s", revision, err) + } + + return cloneDir, nil +} + +// Download the git repo, and create a tarball at the provided commit/tag. +func archiveGitRepo(srcURL, targetDir, version, revision, repo, pkg string, isPkgSubdirInRepo bool, + errPrefix util.ErrPrefix) (string, string, error) { + cloneDir, err := cloneGitRepo(pkg, srcURL, revision) + if err != nil { + return "", "", fmt.Errorf("%s cloning git repo failed: %s", errPrefix, err) + } + + // User should ensure the same fileName is specified in .spec file. + // Format to follow is "-.tar.gz" + // where the version is derived from the revision provided in eext.yaml. + // 1. Tag + // 2. Short Commit Hash + // 3. First 7 charachters of Long Commit Hash + gitArchiveFile := pkg + "-" + version + ".tar.gz" + gitArchiveFilePath := filepath.Join(targetDir, gitArchiveFile) + prefix, err := getRpmNameFromSpecFile(repo, pkg, isPkgSubdirInRepo) + if err != nil { + return "", "", err + } + + // Create the tarball from the specified commit/tag revision + archiveCmd := []string{"archive", + "--prefix", prefix + "/", + "-o", gitArchiveFilePath, + revision, + } + err = util.RunSystemCmdInDir(cloneDir, "git", archiveCmd...) + if err != nil { + return "", "", fmt.Errorf("%sgit archive of %s failed: %s %v", errPrefix, pkg, err, archiveCmd) + } + + return gitArchiveFile, cloneDir, nil +} + // Download the resource srcURL to targetDir // srcURL could be URL or file path // If it is a file:// path, root directory is the diff --git a/impl/create_srpm.go b/impl/create_srpm.go index bdd2f55..eda1f97 100644 --- a/impl/create_srpm.go +++ b/impl/create_srpm.go @@ -23,6 +23,7 @@ type upstreamSrcSpec struct { sigFile string pubKeyPath string skipSigCheck bool + gitSpec util.GitSpec } type srpmBuilder struct { @@ -89,9 +90,13 @@ func (bldr *srpmBuilder) fetchUpstream() error { } for _, upstreamSrcFromManifest := range bldr.pkgSpec.UpstreamSrc { + upstreamUrl := upstreamSrcFromManifest.FullURL + if bldr.pkgSpec.Type == "git" { + upstreamUrl = upstreamSrcFromManifest.GitSpec.Url + } srcParams, err := srcconfig.GetSrcParams( bldr.pkgSpec.Name, - upstreamSrcFromManifest.FullURL, + upstreamUrl, upstreamSrcFromManifest.SourceBundle.Name, upstreamSrcFromManifest.Signature.DetachedSignature.FullURL, upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride, @@ -106,16 +111,43 @@ func (bldr *srpmBuilder) fetchUpstream() error { var downloadErr error upstreamSrc := upstreamSrcSpec{} - bldr.log("downloading %s", srcParams.SrcURL) - // Download source - if upstreamSrc.sourceFile, downloadErr = download( - srcParams.SrcURL, - downloadDir, - repo, pkg, isPkgSubdirInRepo, - bldr.errPrefix); downloadErr != nil { - return downloadErr + if bldr.pkgSpec.Type == "git" { + bldr.log("creating tarball for %s from repo %s", pkg, srcParams.SrcURL) + spec := util.GitSpec{ + SrcUrl: srcParams.SrcURL, + Revision: upstreamSrcFromManifest.GitSpec.Revision, + } + version, err := spec.GetVersionFromRevision() + if err != nil { + return fmt.Errorf("error while getting file version: %s", err) + } + revision := spec.Revision + var clonedDir string + upstreamSrc.sourceFile, clonedDir, downloadErr = archiveGitRepo( + srcParams.SrcURL, + downloadDir, + version, revision, + repo, pkg, isPkgSubdirInRepo, + bldr.errPrefix) + if downloadErr != nil { + return downloadErr + } + + spec.ClonedDir = clonedDir + upstreamSrc.gitSpec = spec + bldr.log("tarball created") + } else { + bldr.log("downloading %s", srcParams.SrcURL) + // Download source + if upstreamSrc.sourceFile, downloadErr = download( + srcParams.SrcURL, + downloadDir, + repo, pkg, isPkgSubdirInRepo, + bldr.errPrefix); downloadErr != nil { + return downloadErr + } + bldr.log("downloaded") } - bldr.log("downloaded") upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey @@ -150,6 +182,25 @@ func (bldr *srpmBuilder) fetchUpstream() error { return fmt.Errorf("%sUnexpected public-key specified for SRPM", bldr.errPrefix) } + } else if bldr.pkgSpec.Type == "git" { + // TODO: Should we enable skip-check by default? + // Skip signature check for git repo if 'signature' field is empty in eext.yaml + // specifiedSignature := (upstreamSrcFromManifest.Signature != manifest.Signature{}) + // skipCheck := upstreamSrcFromManifest.Signature.SkipCheck + // upstreamSrc.skipSigCheck = !specifiedSignature || skipCheck + + if !upstreamSrc.skipSigCheck { + if pubKey == "" { + return fmt.Errorf("%sexpected public-key for %s to verify git repo", + bldr.errPrefix, pkg) + } + pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) + if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { + return fmt.Errorf("%sCannot find public-key at path %s", + bldr.errPrefix, pubKeyPath) + } + upstreamSrc.pubKeyPath = pubKeyPath + } } bldr.upstreamSrc = append(bldr.upstreamSrc, upstreamSrc) @@ -216,6 +267,17 @@ func (bldr *srpmBuilder) verifyUpstream() error { if err := bldr.verifyUpstreamSrpm(); err != nil { return err } + } else if bldr.pkgSpec.Type == "git" { + for _, upstreamSrc := range bldr.upstreamSrc { + if !upstreamSrc.skipSigCheck { + err := util.VerifyGitSignature(upstreamSrc.pubKeyPath, upstreamSrc.gitSpec, bldr.errPrefix) + if err != nil { + return err + } + } + // Deleting cloned git repo since we no longer require it. + os.RemoveAll(upstreamSrc.gitSpec.ClonedDir) + } } else { downloadDir := getDownloadDir(bldr.pkgSpec.Name) for _, upstreamSrc := range bldr.upstreamSrc { @@ -271,7 +333,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeSrpm() error { // also checks tarball signature func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { - supportedTypes := []string{"tarball", "standalone"} + supportedTypes := []string{"tarball", "standalone", "git"} if !slices.Contains(supportedTypes, bldr.pkgSpec.Type) { panic(fmt.Sprintf("%ssetupRpmbuildTreeNonSrpm called for unsupported type %s", bldr.errPrefix, bldr.pkgSpec.Type)) @@ -284,7 +346,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { return err } - if bldr.pkgSpec.Type == "tarball" { + if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "git" { downloadDir := getDownloadDir(bldr.pkgSpec.Name) for _, upstreamSrc := range bldr.upstreamSrc { upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile) @@ -379,7 +441,7 @@ func (bldr *srpmBuilder) setupRpmbuildTree() error { if err := bldr.setupRpmbuildTreeSrpm(); err != nil { return err } - } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" { + } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" || bldr.pkgSpec.Type == "git" { if err := bldr.setupRpmbuildTreeNonSrpm(); err != nil { return err } diff --git a/impl/testData/upstream-git-repo-1/eext.yaml b/impl/testData/upstream-git-repo-1/eext.yaml new file mode 100644 index 0000000..e0c6297 --- /dev/null +++ b/impl/testData/upstream-git-repo-1/eext.yaml @@ -0,0 +1,14 @@ +--- +package: + - name: libpcap + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: libpcap-1.10.1 + signature: + detached-sig: + public-key: tcpdump/tcpdumpPubKey.pem + type: git + build: + repo-bundle: + - name: el9 diff --git a/impl/testData/upstream-git-repo-1/spec/libpcap.spec b/impl/testData/upstream-git-repo-1/spec/libpcap.spec new file mode 100644 index 0000000..01548ce --- /dev/null +++ b/impl/testData/upstream-git-repo-1/spec/libpcap.spec @@ -0,0 +1,174 @@ +%define libpcap_version 1.10.1 + +Name: libpcap +Epoch: 14 +Version: %{libpcap_version} +Release: 1%{?dist}.Ar.1.%{?eext_release:%{eext_release}}%{!?eext_release:eng} +Summary: A system-independent interface for user-level packet capture +Group: Development/Libraries +License: BSD with advertising +URL: http://www.tcpdump.org +BuildRequires: glibc-kernheaders >= 2.2.0 git bison flex libnl3-devel gcc +Requires: libnl3 +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +Source0: libpcap-%libpcap-{libpcap_version}.tar.gz + + + +%description +Libpcap provides a portable framework for low-level network +monitoring. Libpcap can provide network statistics collection, +security monitoring and network debugging. Since almost every system +vendor provides a different interface for packet capture, the libpcap +authors created this system-independent API to ease in porting and to +alleviate the need for several system-dependent packet capture modules +in each application. + +Install libpcap if you need to do low-level network traffic monitoring +on your network. + + +%package devel +Summary: Libraries and header files for the libpcap library +Group: Development/Libraries +Requires: %{name} = %{epoch}:%{version}-%{release} + +%description devel +Libpcap provides a portable framework for low-level network +monitoring. Libpcap can provide network statistics collection, +security monitoring and network debugging. Since almost every system +vendor provides a different interface for packet capture, the libpcap +authors created this system-independent API to ease in porting and to +alleviate the need for several system-dependent packet capture modules +in each application. + +This package provides the libraries, include files, and other +resources needed for developing libpcap applications. + +%prep +%autosetup -S git -n libpcap-%{libpcap_version} + +#sparc needs -fPIC +%ifarch %{sparc} +sed -i -e 's|-fpic|-fPIC|g' configure +%endif + +find . -name '*.c' -o -name '*.h' | xargs chmod 644 + +%build +export CFLAGS="$RPM_OPT_FLAGS -fno-strict-aliasing" +# Explicitly specify each configure flag to avoid dynamically deciding what +# features to include. If we want a feature, ensure that we have the +# supporting libraries listed as BuildRequires/Requires above. +# Read configure.ac to figure out whether it's "--enable/--disable" +# or "--with/--without". +%configure --enable-usb --disable-netmap --without-dpdk --disable-bluetooth \ + --disable-dbus --disable-rdma --without-dag --without-septel --without-snf \ + --without-turbocap +%{?a4_configure:exit 0} +make %{?_smp_mflags} + +%install +rm -rf $RPM_BUILD_ROOT +make install DESTDIR=$RPM_BUILD_ROOT +rm -f $RPM_BUILD_ROOT%{_libdir}/libpcap.a + +%clean +rm -rf $RPM_BUILD_ROOT + +%post -p /sbin/ldconfig + +%postun -p /sbin/ldconfig + + +%files +%defattr(-,root,root) +%doc LICENSE README.md CHANGES CREDITS +%{_libdir}/libpcap.so.* +%{_mandir}/man7/pcap*.7* + +# Listing all include files individually for the benefit of pkgdeps +%files devel +%defattr(-,root,root) +%{_bindir}/pcap-config +%{_includedir}/pcap-bpf.h +%{_includedir}/pcap-namedb.h +%{_includedir}/pcap.h +%{_includedir}/pcap/bluetooth.h +%{_includedir}/pcap/bpf.h +%{_includedir}/pcap/can_socketcan.h +%{_includedir}/pcap/compiler-tests.h +%{_includedir}/pcap/dlt.h +%{_includedir}/pcap/funcattrs.h +%{_includedir}/pcap/ipnet.h +%{_includedir}/pcap/namedb.h +%{_includedir}/pcap/nflog.h +%{_includedir}/pcap/pcap-inttypes.h +%{_includedir}/pcap/pcap.h +%{_includedir}/pcap/sll.h +%{_includedir}/pcap/socket.h +%{_includedir}/pcap/usb.h +%{_includedir}/pcap/vlan.h +%{_libdir}/libpcap.so +%{_libdir}/pkgconfig/libpcap.pc +%{_mandir}/* + + +%changelog +* Fri Apr 22 2011 Miroslav Lichvar 14:1.1.1-3 +- ignore /sys/net/dev files on ENODEV (#693943) +- drop ppp patch +- compile with -fno-strict-aliasing + +* Tue Feb 08 2011 Fedora Release Engineering - 14:1.1.1-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild + +* Tue Apr 06 2010 Miroslav Lichvar 14:1.1.1-1 +- update to 1.1.1 + +* Wed Dec 16 2009 Miroslav Lichvar 14:1.0.0-5.20091201git117cb5 +- update to snapshot 20091201git117cb5 + +* Sat Oct 17 2009 Dennis Gilmore 14:1.0.0-4.20090922gite154e2 +- use -fPIC on sparc arches + +* Wed Sep 23 2009 Miroslav Lichvar 14:1.0.0-3.20090922gite154e2 +- update to snapshot 20090922gite154e2 +- drop old soname + +* Fri Jul 24 2009 Fedora Release Engineering - 14:1.0.0-2.20090716git6de2de +- Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild + +* Wed Jul 22 2009 Miroslav Lichvar 14:1.0.0-1.20090716git6de2de +- update to 1.0.0, git snapshot 20090716git6de2de + +* Wed Feb 25 2009 Fedora Release Engineering - 14:0.9.8-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild + +* Fri Jun 27 2008 Miroslav Lichvar 14:0.9.8-3 +- use CFLAGS when linking (#445682) + +* Tue Feb 19 2008 Fedora Release Engineering - 14:0.9.8-2 +- Autorebuild for GCC 4.3 + +* Wed Oct 24 2007 Miroslav Lichvar 14:0.9.8-1 +- update to 0.9.8 + +* Wed Aug 22 2007 Miroslav Lichvar 14:0.9.7-3 +- update license tag + +* Wed Jul 25 2007 Jesse Keating - 14:0.9.7-2 +- Rebuild for RH #249435 + +* Tue Jul 24 2007 Miroslav Lichvar 14:0.9.7-1 +- update to 0.9.7 + +* Tue Jun 19 2007 Miroslav Lichvar 14:0.9.6-1 +- update to 0.9.6 + +* Tue Nov 28 2006 Miroslav Lichvar 14:0.9.5-1 +- split from tcpdump package (#193657) +- update to 0.9.5 +- don't package static library +- maintain soname diff --git a/impl/upstream_git_test.go b/impl/upstream_git_test.go new file mode 100644 index 0000000..5df3968 --- /dev/null +++ b/impl/upstream_git_test.go @@ -0,0 +1,201 @@ +// Copyright (c) 2023 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package impl + +import ( + "os" + "path/filepath" + "testing" + + "code.arista.io/eos/tools/eext/util" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +type TestDataType struct { + gitSpec *util.GitSpec + expectedValue string +} + +func populateTestData(cloneDir string, revisionList, expectedList []string) []*TestDataType { + srcURL := "https://github.com/the-tcpdump-group/libpcap" + + var dataList []*TestDataType + for i, revision := range revisionList { + gitSpec := &util.GitSpec{ + SrcUrl: srcURL, + Revision: revision, + ClonedDir: cloneDir, + } + dataType := &TestDataType{ + gitSpec: gitSpec, + expectedValue: expectedList[i], + } + dataList = append(dataList, dataType) + } + + return dataList +} + +func populateTestDataForRevision(cloneDir string) []*TestDataType { + revisionList := []string{"libpcap-1.10.4", "95691eb", "59747a7e74506bd2fbf6cc668e1d66b68ac6eb6d"} + expectedList := []string{"TAG", "COMMIT", "COMMIT"} + + // Set empty git repo for some entries + testData := populateTestData(cloneDir, revisionList, expectedList) + for i, data := range testData { + if i%2 == 0 { + data.gitSpec.ClonedDir = "" + } + } + + return testData +} + +func populateTestDataForVersion(cloneDir string) []*TestDataType { + revisionList := []string{"libpcap-1.10.4", "49c28589076576b6cdc3772e76c59b5b7ded78a1", "59747a7"} + expectedList := []string{"libpcap-1.10.4", "49c2858", "59747a7"} + + return populateTestData(cloneDir, revisionList, expectedList) +} + +func populateTestDataForGitSignature(cloneDir string) []*TestDataType { + // Yet to verify commit signatures, + // since not many commits signed with public keys are available. + revisionList := []string{"libpcap-1.10.1"} + expectedList := []string{""} + + return populateTestData(cloneDir, revisionList, expectedList) +} + +func cloneGitDir() (string, error) { + srcURL := "https://github.com/the-tcpdump-group/libpcap" + revision := "380ab16f9ec0958a5fbaa2da0240a63ed3222335" + pkg := "libpcap" + + cloneDir, err := cloneGitRepo(pkg, srcURL, revision) + if err != nil { + return "", err + } + + return cloneDir, nil +} + +func TestRevisionType(t *testing.T) { + cloneDir, err := cloneGitDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(cloneDir) + + testDataList := populateTestDataForRevision(cloneDir) + for _, data := range testDataList { + gitSpec := data.gitSpec + expectedType := data.expectedValue + + typeLocalRepo, err := gitSpec.TypeOfGitRevision() + if err != nil { + t.Fatal(err) + } + typeRemoteRepo, err := gitSpec.TypeOfGitRevisionFromRemote() + if err != nil { + t.Fatal(err) + } + + require.Equal(t, expectedType, typeLocalRepo) + require.Equal(t, expectedType, typeRemoteRepo) + } + t.Log("Test typeOfGitRevision PASSED") +} + +func TestVersionFromRevision(t *testing.T) { + cloneDir := "" + testDataList := populateTestDataForVersion(cloneDir) + + for _, data := range testDataList { + gitSpec := data.gitSpec + expectedType := data.expectedValue + + version, err := gitSpec.GetVersionFromRevision() + if err != nil { + t.Fatal(err) + } + + require.Equal(t, expectedType, version) + } + t.Log("Test versionFromRevision PASSED") +} + +func TestRpmNameFromSpecFile(t *testing.T) { + viper.Set("SrcDir", "./testData/") + defer viper.Reset() + pkg := "libpcap" + repo := "upstream-git-repo-1" + expectedRpmName := "libpcap-1.10.1" + + gotRpmName, err := getRpmNameFromSpecFile(repo, pkg, false) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, expectedRpmName, gotRpmName) + t.Log("Test rpmNameFromSpecFile PASSED") +} + +func TestVerifyGitSignature(t *testing.T) { + cloneDir, err := cloneGitDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(cloneDir) + + viper.Set("PkiPath", "../pki") + defer viper.Reset() + pubKeyPath := filepath.Join(getDetachedSigDir(), "tcpdump/tcpdumpPubKey.pem") + testData := populateTestDataForGitSignature(cloneDir) + for _, data := range testData { + gitSpec := data.gitSpec + + err = util.VerifyGitSignature(pubKeyPath, *gitSpec, "") + if err != nil { + t.Fatal(err) + } + } + t.Log("Test verifyGitRepoSignature PASSED") +} + +func TestGitArchive(t *testing.T) { + t.Log("Create temporary working directory") + testWorkingDir, mkdirErr := os.MkdirTemp("", "upstream-git") + if mkdirErr != nil { + t.Fatal(mkdirErr) + } + defer os.RemoveAll(testWorkingDir) + + srcURL := "https://github.com/the-tcpdump-group/libpcap" + version := "libpcap-1.10.1" + revision := "libpcap-1.10.1" + + viper.Set("SrcDir", "./testData/") + defer viper.Reset() + pkg := "libpcap" + repo := "upstream-git-repo-1" + + archiveFile, cloneDir, err := archiveGitRepo(srcURL, testWorkingDir, version, revision, repo, pkg, false, "") + if err != nil { + t.Fatal(err) + } + + archivePath := filepath.Join(testWorkingDir, archiveFile) + err = util.CheckPath(archivePath, false, false) + if err != nil { + t.Fatal(err) + } + + os.RemoveAll(cloneDir) + os.RemoveAll(testWorkingDir) + + t.Log("Test gitArchive PASSED") +} diff --git a/manifest/manifest.go b/manifest/manifest.go index afabb5a..960e751 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -5,7 +5,7 @@ package manifest import ( "fmt" - "io/ioutil" + "os" "path/filepath" "golang.org/x/exp/slices" @@ -118,12 +118,18 @@ type SourceBundle struct { SrcRepoParamsOverride srcconfig.SrcRepoParamsOverride `yaml:"override"` } +type GitSpec struct { + Url string `yaml:"url"` + Revision string `yaml:"revision"` +} + // UpstreamSrc spec // Lists each source bundle(tarball/srpm) and // detached signature file for tarball. type UpstreamSrc struct { SourceBundle SourceBundle `yaml:"source-bundle"` FullURL string `yaml:"full-url"` + GitSpec GitSpec `yaml:"git"` Signature Signature `yaml:"signature"` } @@ -147,7 +153,7 @@ type Manifest struct { } func (m Manifest) sanityCheck() error { - allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone"} + allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone", "git"} for _, pkgSpec := range m.Package { if pkgSpec.Name == "" { @@ -184,23 +190,49 @@ func (m Manifest) sanityCheck() error { } for _, upStreamSrc := range pkgSpec.UpstreamSrc { - specifiedFullSrcURL := (upStreamSrc.FullURL != "") - specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) - if !specifiedFullSrcURL && !specifiedSrcBundle { - return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + if pkgSpec.Type == "git" { + specifiedUrl := (upStreamSrc.GitSpec.Url != "") + specifiedRevision := (upStreamSrc.GitSpec.Revision != "") + if !specifiedUrl { + return fmt.Errorf("please provide the url for git repo of package %s", pkgSpec.Name) + } + if !specifiedRevision { + return fmt.Errorf("please provide a commit/tag to define revision of package %s", pkgSpec.Name) + } - if specifiedFullSrcURL && specifiedSrcBundle { - return fmt.Errorf( - "Conflicting sources for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + specifiedSignature := (upStreamSrc.Signature != Signature{}) + if specifiedSignature { + skipSigCheck := (upStreamSrc.Signature.SkipCheck) + specifiedPubKey := (upStreamSrc.Signature.DetachedSignature.PubKey != "") + if !skipSigCheck && !specifiedPubKey { + return fmt.Errorf( + "please provide the public key to verify git repo for package %s, or skip signature check", + pkgSpec.Name) + } + } else { + return fmt.Errorf( + "signature fields not specified for package %s, provide public key or skip signature check", + pkgSpec.Name) + } + } else { + specifiedFullSrcURL := (upStreamSrc.FullURL != "") + specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) + if !specifiedFullSrcURL && !specifiedSrcBundle { + return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } - specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" - if specifiedFullSigURL && specifiedSrcBundle { - return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", - pkgSpec.Name) + if specifiedFullSrcURL && specifiedSrcBundle { + return fmt.Errorf( + "Conflicting sources for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } + + specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" + if specifiedFullSigURL && specifiedSrcBundle { + return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", + pkgSpec.Name) + } } } } @@ -213,9 +245,9 @@ func LoadManifest(repo string) (*Manifest, error) { repoDir := util.GetRepoDir(repo) yamlPath := filepath.Join(repoDir, "eext.yaml") - yamlContents, readErr := ioutil.ReadFile(yamlPath) + yamlContents, readErr := os.ReadFile(yamlPath) if readErr != nil { - return nil, fmt.Errorf("manifest.LoadManifest: ioutil.ReadFile on %s returned %s", yamlPath, readErr) + return nil, fmt.Errorf("manifest.LoadManifest: os.ReadFile on %s returned %s", yamlPath, readErr) } var manifest Manifest diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 45bb33c..f417a0e 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -31,12 +31,15 @@ func TestManifest(t *testing.T) { viper.Set("SrcDir", dir) defer viper.Reset() - t.Log("Copy sample manifest to test directory") - testutil.SetupManifest(t, dir, "pkg1", "sampleManifest1.yaml") + testFiles := []string{"sampleManifest1.yaml", "sampleManifest4.yaml"} + for _, testFile := range testFiles { + t.Logf("Copy sample manifest %s to test directory", testFile) + testutil.SetupManifest(t, dir, "pkg1", testFile) - t.Log("Testing Load") - testLoad(t, "pkg1") - t.Log("Load test passed") + t.Log("Testing Load") + testLoad(t, "pkg1") + t.Log("Load test passed") + } } type manifestTestVariant struct { @@ -57,16 +60,21 @@ func TestManifestNegative(t *testing.T) { defer viper.Reset() testCases := map[string]manifestTestVariant{ - "testBundleAndFullURL": manifestTestVariant{ + "testBundleAndFullURL": { TestPkg: "pkg2", ManifestFile: "sampleManifest2.yaml", ExpectedErr: "Conflicting sources for Build in package libpcap, provide either full-url or source-bundle", }, - "testBundleAndSignature": manifestTestVariant{ + "testBundleAndSignature": { TestPkg: "pkg3", ManifestFile: "sampleManifest3.yaml", ExpectedErr: "Conflicting signatures for Build in package tcpdump, provide full-url or source-bundle", }, + "testGitUpstreamWithoutSignature": { + TestPkg: "pkg5", + ManifestFile: "sampleManifest5.yaml", + ExpectedErr: "signature fields not specified for package libpcap, provide public key or skip signature check", + }, } for testName, variant := range testCases { t.Logf("%s: Copy sample manifest to test directory", testName) diff --git a/manifest/testData/sampleManifest4.yaml b/manifest/testData/sampleManifest4.yaml new file mode 100644 index 0000000..e11b00a --- /dev/null +++ b/manifest/testData/sampleManifest4.yaml @@ -0,0 +1,30 @@ +--- +package: + - name: libpcap1 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: 104271ba4a14de6743e43bcf87536786d8fddea4 + signature: + detached-sig: + public-key: mrtparse/mrtparsePubKey.pem + type: git + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + + - name: libpcap2 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: libpcap-1.10.1 + signature: + skip-check: true + type: git + build: + repo-bundle: + - name: foo + version: v1 + - name: bar diff --git a/manifest/testData/sampleManifest5.yaml b/manifest/testData/sampleManifest5.yaml new file mode 100644 index 0000000..392e1cc --- /dev/null +++ b/manifest/testData/sampleManifest5.yaml @@ -0,0 +1,13 @@ +--- +package: + - name: libpcap + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: 104271ba4a14de6743e43bcf87536786d8fddea4 + type: git + build: + repo-bundle: + - name: foo + version: v1 + - name: bar diff --git a/srcconfig/srcconfig.go b/srcconfig/srcconfig.go index f63d3b1..d3d5eb0 100644 --- a/srcconfig/srcconfig.go +++ b/srcconfig/srcconfig.go @@ -6,7 +6,6 @@ package srcconfig import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -222,9 +221,9 @@ func LoadSrcConfig() (*SrcConfig, error) { cfgPath, statErr) } - yamlContents, readErr := ioutil.ReadFile(cfgPath) + yamlContents, readErr := os.ReadFile(cfgPath) if readErr != nil { - return nil, fmt.Errorf("srcconfig.LoadSrcConfig: ioutil.ReadFile on %s returned %s", + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: os.ReadFile on %s returned %s", cfgPath, readErr) } diff --git a/testutil/testutil.go b/testutil/testutil.go index dde6779..532510a 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -5,7 +5,7 @@ package testutil import ( "fmt" - "io/ioutil" + "io" "os" "os/exec" "path/filepath" @@ -191,7 +191,7 @@ func setupQuiet() { func checkAndCleanupQuiet(t *testing.T) { w.Close() - out, err := ioutil.ReadAll(r) + out, err := io.ReadAll(r) if err != nil { t.Fatal(err) } diff --git a/util/util.go b/util/util.go index b4012d0..9f14298 100644 --- a/util/util.go +++ b/util/util.go @@ -4,6 +4,7 @@ package util import ( + "bytes" "fmt" "io" "os" @@ -26,6 +27,80 @@ var GlobalVar Globals // ErrPrefix is a container type for error prefix strings. type ErrPrefix string +type GitSpec struct { + SrcUrl string + Revision string + ClonedDir string +} + +// Returns if the provided revision is a "COMMIT" or a "TAG" +func (spec *GitSpec) TypeOfGitRevision() (string, error) { + repoPath := spec.ClonedDir + revision := spec.Revision + if err := CheckPath(repoPath, true, false); err != nil { + return spec.TypeOfGitRevisionFromRemote() + } + + cmd := exec.Command("git", "show", revision) + cmd.Dir = repoPath + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git show of repo %s failed", repoPath) + } + + topLine := strings.SplitAfterN(out.String(), "\n", 2)[0] + if !strings.Contains(topLine, revision) { + return "", fmt.Errorf("revision %s not found in repo %s, please provide valid commit/tag", revision, repoPath) + } + + tagFormat := "tag " + revision + if strings.Contains(topLine, tagFormat) { + return "TAG", nil + } else { + return "COMMIT", nil + } +} + +func (spec *GitSpec) TypeOfGitRevisionFromRemote() (string, error) { + srcURL := spec.SrcUrl + revision := spec.Revision + cmd := exec.Command("git", "ls-remote", srcURL, revision) + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git ls-remote of repo %s failed", srcURL) + } + if len(out.String()) == 0 { + return "COMMIT", nil + } else if strings.Contains(out.String(), revision) { + return "TAG", nil + } else { + return "", fmt.Errorf("revision %s not found in repo %s, please provide valid commit/tag", revision, srcURL) + } +} + +// Returns a unique version number based on the commit/tag +func (spec *GitSpec) GetVersionFromRevision() (string, error) { + revision := spec.Revision + typeRevision, err := spec.TypeOfGitRevision() + if err != nil { + return "", err + } + + if typeRevision == "TAG" { + return revision, nil + } + + // If short commit hash is provided, return as is. + // For long commit hash, return the first 7 charachters. + if len(revision) <= 7 { + return revision, nil + } else { + return revision[:7], nil + } +} + // RunSystemCmd runs a command on the shell and pipes to stdout and stderr func RunSystemCmd(name string, arg ...string) error { cmd := exec.Command(name, arg...) @@ -39,6 +114,20 @@ func RunSystemCmd(name string, arg ...string) error { return err } +// Runs the system command from a specified directory +func RunSystemCmdInDir(dir string, name string, arg ...string) error { + cmd := exec.Command(name, arg...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + if !GlobalVar.Quiet { + cmd.Stdout = os.Stdout + } else { + cmd.Stdout = io.Discard + } + err := cmd.Run() + return err +} + // CheckOutput runs a command on the shell and returns stdout if it is successful // else it return the error func CheckOutput(name string, arg ...string) ( @@ -48,11 +137,11 @@ func CheckOutput(name string, arg ...string) ( if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return string(output), - fmt.Errorf("Running '%s %s': exited with exit-code %d\nstderr:\n%s", + fmt.Errorf("running '%s %s': exited with exit-code %d\nstderr:\n%s", name, strings.Join(arg, " "), exitErr.ExitCode(), exitErr.Stderr) } return string(output), - fmt.Errorf("Running '%s %s' failed with '%s'", + fmt.Errorf("running '%s %s' failed with '%s'", name, strings.Join(arg, " "), err) } return string(output), nil @@ -204,3 +293,52 @@ func VerifyTarballSignature( return nil } + +// VerifyGitSignature verifies that the git repo commit/tag is signed. +func VerifyGitSignature(pubKeyPath string, gitSpec GitSpec, errPrefix ErrPrefix) error { + tmpDir, mkdtErr := os.MkdirTemp("", "eext-keyring") + if mkdtErr != nil { + return fmt.Errorf("%sError '%s'creating temp dir for keyring", + errPrefix, mkdtErr) + } + defer os.RemoveAll(tmpDir) + + err := os.Setenv("GNUPGHOME", tmpDir) + if err != nil { + return fmt.Errorf("%sunable to set ENV variable GNUPGHOME", errPrefix) + } + defer os.Unsetenv("GNUPGHOME") + + if err := RunSystemCmd("gpg", "--fingerprint"); err != nil { + return fmt.Errorf("%sError '%s'creating keyring", + errPrefix, err) + } + + // Import public key + if err := RunSystemCmd("gpg", "--import", pubKeyPath); err != nil { + return fmt.Errorf("%sError '%s' importing public-key %s", + errPrefix, err, pubKeyPath) + } + + var verifyRepoCmd []string + revision := gitSpec.Revision + revisionType, err := gitSpec.TypeOfGitRevision() + if err != nil { + return fmt.Errorf("%sinvalid revision %s provided, provide either a COMMIT or TAG %s", errPrefix, revision, err) + } + if revisionType == "COMMIT" { + verifyRepoCmd = []string{"verify-commit", "-v", revision} + } else if revisionType == "TAG" { + verifyRepoCmd = []string{"verify-tag", "-v", revision} + } else { + return fmt.Errorf("%sinvalid revision %s provided, provide either a COMMIT or TAG", errPrefix, revision) + } + + clonedDir := gitSpec.ClonedDir + err = RunSystemCmdInDir(clonedDir, "git", verifyRepoCmd...) + if err != nil { + return fmt.Errorf("%serror during verifying git repo at %s: %s", errPrefix, clonedDir, err) + } + + return nil +}