From 16f815552582e2caa9d61a55ffa5f7f7eee130be Mon Sep 17 00:00:00 2001 From: Manish Kumar Singh Date: Tue, 10 Dec 2024 04:38:17 +0000 Subject: [PATCH] Add `list-unverified-sources` root command and SHA-256 hash support for unverified upstream sources This new root command is part of the `stest` and is designed to list all upstream sources with the `skip-check` flag set to `true` for a specified package. The output is written to a file at the following location: `/dest/code.arista.io/eos/eext/{package}.unverifiedSources.json`. This file will be included in the snapshot build created by Barney, ensuring better tracking of unverified sources. Unverified upstream sources (those with `skip-check: true`) are inherently risky as they lack a verification method. Any changes to these sources could go undetected. To address this, we now calculate and store the SHA-256 hash of unverified sources. This hash is added to the `eext.yaml` file under the `src-sha256-hash` field. During the `create-srpm` command, the hash in `eext.yaml` will be compared with the hash of the downloaded sources. --- cmd/list_unverified_sources.go | 29 ++++++++++ go.sum | 6 ++ impl/create_srpm.go | 41 ++++++++++++++ impl/create_srpm_from_others_test.go | 5 ++ impl/list_unverified_sources.go | 78 ++++++++++++++++++++++++++ impl/list_unverified_sources_test.go | 33 +++++++++++ impl/testData/unverified-src/eext.yaml | 25 +++++++++ manifest/manifest.go | 1 + 8 files changed, 218 insertions(+) create mode 100644 cmd/list_unverified_sources.go create mode 100644 impl/list_unverified_sources.go create mode 100644 impl/list_unverified_sources_test.go create mode 100644 impl/testData/unverified-src/eext.yaml diff --git a/cmd/list_unverified_sources.go b/cmd/list_unverified_sources.go new file mode 100644 index 0000000..15bf996 --- /dev/null +++ b/cmd/list_unverified_sources.go @@ -0,0 +1,29 @@ +// Copyright (c) 2022 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package cmd + +import ( + "code.arista.io/eos/tools/eext/impl" + "github.com/spf13/cobra" +) + +// listUnverifiedSourcescmd represents the list-unverified-sources command +var listUnverifiedSourcescmd = &cobra.Command{ + Use: "list-unverified-sources", + Short: "list unverified upstream sources", + Long: `Checks for the upstream sources within package which don't have a valid signature check i.e, skip-check flag is true + and generates content hash for the upstream sources.`, + RunE: func(cmd *cobra.Command, args []string) error { + repo, _ := cmd.Flags().GetString("repo") + pkg, _ := cmd.Flags().GetString("package") + err := impl.ListUnverifiedSources(repo, pkg) + return err + }, +} + +func init() { + listUnverifiedSourcescmd.Flags().StringP("repo", "r", "", "Repository name (OPTIONAL)") + listUnverifiedSourcescmd.Flags().StringP("package", "p", "", "specify package name (OPTIONAL)") + rootCmd.AddCommand(listUnverifiedSourcescmd) +} diff --git a/go.sum b/go.sum index f5b353c..b2b6d1c 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= @@ -310,6 +312,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -319,6 +323,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/impl/create_srpm.go b/impl/create_srpm.go index ebf4ee0..c9421b7 100644 --- a/impl/create_srpm.go +++ b/impl/create_srpm.go @@ -4,7 +4,9 @@ package impl import ( + "crypto/sha256" "fmt" + "io" "log" "os" "path/filepath" @@ -74,6 +76,27 @@ func (bldr *srpmBuilder) clean() error { return nil } +// Generate SHA256 hash of the downloaded upstream source +// and match it with the SHA256 hash in eext.yaml +func generateSrcSha256Hash(srcFilePath string) (string, error) { + + srcFile, err := os.Open(srcFilePath) + if err != nil { + return "", fmt.Errorf("errored with %s while creating file", + err) + } + defer srcFile.Close() + + hashComputer := sha256.New() + if _, err := io.Copy(hashComputer, srcFile); err != nil { + return "", fmt.Errorf("errored with %s while generating hash", + err) + } + sha256Hash := fmt.Sprintf("%x", hashComputer.Sum(nil)) + + return sha256Hash, nil +} + // Fetch the upstream sources mentioned in the manifest. // Put them into downloadDir and populate bldr.upstreamSrc func (bldr *srpmBuilder) fetchUpstream() error { @@ -98,6 +121,24 @@ func (bldr *srpmBuilder) fetchUpstream() error { if err != nil { return err } + + if upstreamSrcFromManifest.Signature.SkipCheck && upstreamSrcFromManifest.Signature.SrcSha256Hash != "" { + srcFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile) + sha256Hash, err := generateSrcSha256Hash(srcFilePath) + if err != nil { + return fmt.Errorf("%sError '%s'", + bldr.errPrefix, err) + } + eextSha256Hash := upstreamSrcFromManifest.Signature.SrcSha256Hash + fmt.Printf("calculated SHA hash is `%s`, hash in eext-yaml file is `%s` \n", sha256Hash, eextSha256Hash) + if sha256Hash != eextSha256Hash { + return fmt.Errorf("%sError:SHA256 hash '%s'is not matching with eext.yaml sha hash '%s' for upstream file '%s', package '%s'", + bldr.errPrefix, sha256Hash, eextSha256Hash, srcFilePath, bldr.pkgSpec.Name) + } else { + fmt.Printf("SHA-256 hash matched successfully, unmodified upstream source found \n") + } + } + bldr.upstreamSrc = append(bldr.upstreamSrc, *upstreamSrc) } diff --git a/impl/create_srpm_from_others_test.go b/impl/create_srpm_from_others_test.go index 92981df..db01295 100644 --- a/impl/create_srpm_from_others_test.go +++ b/impl/create_srpm_from_others_test.go @@ -1,3 +1,8 @@ +// Copyright (c) 2024 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +//go:build containerized + package impl import ( diff --git a/impl/list_unverified_sources.go b/impl/list_unverified_sources.go new file mode 100644 index 0000000..1f8c1ed --- /dev/null +++ b/impl/list_unverified_sources.go @@ -0,0 +1,78 @@ +// Copyright (c) 2022 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package impl + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "code.arista.io/eos/tools/eext/manifest" + "code.arista.io/eos/tools/eext/util" +) + +// ListUnverifiedSources lists all the upstream sources within a package +// which do not have valid signature check. For The upstream sources with +// `skip-check` flag as true content hash is generated +func ListUnverifiedSources(repo string, pkg string) error { + + // load the eext yaml + repoManifest, loadManifestErr := manifest.LoadManifest(repo) + if loadManifestErr != nil { + return loadManifestErr + } + curPath, _ := os.Getwd() + splittedCurPath := strings.Split(curPath, "/") + repoName := splittedCurPath[len(splittedCurPath)-1] + + var checkAllPackages bool = (pkg == "") + + // check for skip-check flag in thr manifest + for _, pkgSpec := range repoManifest.Package { + thisPkgName := pkgSpec.Name + + if !checkAllPackages && thisPkgName != pkg { + continue + } + errPrefix := util.ErrPrefix(fmt.Sprintf("listUnverifiedSources(%s)", thisPkgName)) + upstreamSources := []manifest.UpstreamSrc{} + + for _, upstreamSrcFromManifest := range pkgSpec.UpstreamSrc { + if !upstreamSrcFromManifest.Signature.SkipCheck { + continue + } + upstreamSources = append(upstreamSources, upstreamSrcFromManifest) + } + + if len(upstreamSources) == 0 { + return nil + } + + JsonUpstreamSrcHashes, err := json.MarshalIndent(upstreamSources, "", " ") + if err != nil { + return fmt.Errorf("%s unable to convert map to json \n errored with %s ", + errPrefix, err) + } + + upstreamInfoFile := fmt.Sprintf("/dest/code.arista.io/eos/eext/%s/%s/unVerifiedSources.json", repoName, thisPkgName) + upstreamInfoDir := filepath.Dir(upstreamInfoFile) + if err := os.MkdirAll(upstreamInfoDir, 0755); err != nil { + return fmt.Errorf("%s unable to create empty dir path \n errored with %s ", + errPrefix, err) + } + + if err := os.WriteFile(upstreamInfoFile, JsonUpstreamSrcHashes, 0777); err != nil { + return fmt.Errorf("%s unable to write to file \n errored with %s ", + errPrefix, err) + } + } + + // sudo eext list-unverified-sources -p pkg + // if skip-check is true download the upstream source + // calculate the sha-256 hash for the upstream source tarball + + return nil +} diff --git a/impl/list_unverified_sources_test.go b/impl/list_unverified_sources_test.go new file mode 100644 index 0000000..f57b279 --- /dev/null +++ b/impl/list_unverified_sources_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2023 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +//go:build containerized + +package impl + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func checkFileExists(filePath string) error { + _, err := os.Stat(filePath) + return err +} + +func TestListUnverifiedSources(t *testing.T) { + curPath, _ := os.Getwd() + repo := filepath.Join(curPath, "testData/unverified-src") + + ListUnverifiedSources(repo, "foo1") + filePath := "/dest/code.arista.io/eos/eext/foo1/unVerifiedSources.json" + require.NotEqual(t, nil, checkFileExists(filePath)) + + ListUnverifiedSources(repo, "foo2") + filePath = "/dest/code.arista.io/eos/eext/foo2/unVerifiedSources.json" + require.Equal(t, nil, checkFileExists(filePath)) + t.Log("TestListUnverifiedSources test Passed") +} diff --git a/impl/testData/unverified-src/eext.yaml b/impl/testData/unverified-src/eext.yaml new file mode 100644 index 0000000..8b602cd --- /dev/null +++ b/impl/testData/unverified-src/eext.yaml @@ -0,0 +1,25 @@ +--- +package: + - name: foo1 + upstream-sources: + - source-bundle: + name: srpm + override: + version: 1.7.7-1.fc40 + type: srpm + build: + repo-bundle: + - name: el9 + + - name: foo2 + upstream-sources: + - source-bundle: + name: srpm + override: + version: 1.7.7-1.fc40 + signature: + skip-check: true + type: srpm + build: + repo-bundle: + - name: el9 \ No newline at end of file diff --git a/manifest/manifest.go b/manifest/manifest.go index 5296447..7739abf 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -109,6 +109,7 @@ type DetachedSignature struct { type Signature struct { SkipCheck bool `yaml:"skip-check"` DetachedSignature DetachedSignature `yaml:"detached-sig"` + SrcSha256Hash string `yaml:"src-sha256-hash"` } // SourceBundle spec