Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Track supporting DPKG evidence #3228

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Track supporting DPKG evidence #3228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions internal/relationship/evident_by.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,18 @@ func EvidentBy(catalog *pkg.Collection) []artifact.Relationship {
var edges []artifact.Relationship
for _, p := range catalog.Sorted() {
for _, l := range p.Locations.ToSlice() {
if v, exists := l.Annotations[pkg.EvidenceAnnotationKey]; !exists || v != pkg.PrimaryEvidenceAnnotation {
// skip non-primary evidence from being expressed as a relationship.
// note: this may be configurable in the future.
continue
kind := pkg.SupportingEvidenceAnnotation
if v, exists := l.Annotations[pkg.EvidenceAnnotationKey]; exists {
kind = v
}

edges = append(edges, artifact.Relationship{
From: p,
To: l.Coordinates,
Type: artifact.EvidentByRelationship,
Data: map[string]string{
"kind": kind,
},
})
}
}
Expand Down
16 changes: 12 additions & 4 deletions internal/task/file_tasks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,10 @@ func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accesso
}

if selection == file.FilesOwnedByPackageSelection {
var coordinates []file.Coordinates
var coordinates file.CoordinateSet

accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
// get any file coordinates that are owned by a package
for _, r := range sbom.Relationships {
if r.Type != artifact.ContainsRelationship {
continue
Expand All @@ -145,16 +146,23 @@ func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accesso
continue
}
if c, ok := r.To.(file.Coordinates); ok {
coordinates = append(coordinates, c)
coordinates.Add(c)
}
}

// get any file coordinates referenced by a package directly
for p := range sbom.Artifacts.Packages.Enumerate() {
coordinates.Add(p.Locations.CoordinateSet().ToSlice()...)
}
})

if len(coordinates) == 0 {
coords := coordinates.ToSlice()

if len(coords) == 0 {
return nil, false
}

return coordinates, true
return coords, true
}

return nil, false
Expand Down
28 changes: 15 additions & 13 deletions syft/pkg/cataloger/debian/cataloger_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,16 @@ func TestDpkgCataloger(t *testing.T) {
Version: "1.1.8-3.6",
FoundBy: "dpkg-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("GPL-1", file.NewVirtualLocation("/usr/share/doc/libpam-runtime/copyright", "/usr/share/doc/libpam-runtime/copyright")),
pkg.NewLicenseFromLocations("GPL-2", file.NewVirtualLocation("/usr/share/doc/libpam-runtime/copyright", "/usr/share/doc/libpam-runtime/copyright")),
pkg.NewLicenseFromLocations("LGPL-2.1", file.NewVirtualLocation("/usr/share/doc/libpam-runtime/copyright", "/usr/share/doc/libpam-runtime/copyright")),
pkg.NewLicenseFromLocations("GPL-1", file.NewLocation("/usr/share/doc/libpam-runtime/copyright")),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you help me understand why this stopped being a virtual location and became a regular location? Is this change visible to users in the eventual SBOM?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was cleanup -- since I was in this file and making new Locations, I noticed that all params to the original call were the same, which is the same as using the NewLocation constructor. So I made the remaining parts of the file that were using this function needlessly as consistent as the new code I was adding.

pkg.NewLicenseFromLocations("GPL-2", file.NewLocation("/usr/share/doc/libpam-runtime/copyright")),
pkg.NewLicenseFromLocations("LGPL-2.1", file.NewLocation("/usr/share/doc/libpam-runtime/copyright")),
),
Locations: file.NewLocationSet(
file.NewVirtualLocation("/var/lib/dpkg/status", "/var/lib/dpkg/status"),
file.NewVirtualLocation("/var/lib/dpkg/info/libpam-runtime.md5sums", "/var/lib/dpkg/info/libpam-runtime.md5sums"),
file.NewVirtualLocation("/var/lib/dpkg/info/libpam-runtime.conffiles", "/var/lib/dpkg/info/libpam-runtime.conffiles"),
file.NewVirtualLocation("/usr/share/doc/libpam-runtime/copyright", "/usr/share/doc/libpam-runtime/copyright"),
file.NewLocation("/var/lib/dpkg/status"),
file.NewLocation("/var/lib/dpkg/info/libpam-runtime.preinst"),
file.NewLocation("/var/lib/dpkg/info/libpam-runtime.md5sums"),
file.NewLocation("/var/lib/dpkg/info/libpam-runtime.conffiles"),
file.NewLocation("/usr/share/doc/libpam-runtime/copyright"),
),
Type: pkg.DebPkg,
Metadata: pkg.DpkgDBEntry{
Expand Down Expand Up @@ -98,14 +99,15 @@ func TestDpkgCataloger(t *testing.T) {
Version: "3.34.1-3",
FoundBy: "dpkg-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("public-domain", file.NewVirtualLocation("/usr/share/doc/libsqlite3-0/copyright", "/usr/share/doc/libsqlite3-0/copyright")),
pkg.NewLicenseFromLocations("GPL-2+", file.NewVirtualLocation("/usr/share/doc/libsqlite3-0/copyright", "/usr/share/doc/libsqlite3-0/copyright")),
pkg.NewLicenseFromLocations("GPL-2", file.NewVirtualLocation("/usr/share/doc/libsqlite3-0/copyright", "/usr/share/doc/libsqlite3-0/copyright")),
pkg.NewLicenseFromLocations("public-domain", file.NewLocation("/usr/share/doc/libsqlite3-0/copyright")),
pkg.NewLicenseFromLocations("GPL-2+", file.NewLocation("/usr/share/doc/libsqlite3-0/copyright")),
pkg.NewLicenseFromLocations("GPL-2", file.NewLocation("/usr/share/doc/libsqlite3-0/copyright")),
),
Locations: file.NewLocationSet(
file.NewVirtualLocation("/var/lib/dpkg/status.d/libsqlite3-0", "/var/lib/dpkg/status.d/libsqlite3-0"),
file.NewVirtualLocation("/var/lib/dpkg/status.d/libsqlite3-0.md5sums", "/var/lib/dpkg/status.d/libsqlite3-0.md5sums"),
file.NewVirtualLocation("/usr/share/doc/libsqlite3-0/copyright", "/usr/share/doc/libsqlite3-0/copyright"),
file.NewLocation("/var/lib/dpkg/status.d/libsqlite3-0"),
file.NewLocation("/var/lib/dpkg/status.d/libsqlite3-0.md5sums"),
file.NewLocation("/var/lib/dpkg/status.d/libsqlite3-0.preinst"),
file.NewLocation("/usr/share/doc/libsqlite3-0/copyright"),
),
Type: pkg.DebPkg,
Metadata: pkg.DpkgDBEntry{
Expand Down
18 changes: 11 additions & 7 deletions syft/pkg/cataloger/debian/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,18 @@ const (
docsPath = "/usr/share/doc"
)

func newDpkgPackage(d pkg.DpkgDBEntry, dbLocation file.Location, resolver file.Resolver, release *linux.Release) pkg.Package {
func newDpkgPackage(d pkg.DpkgDBEntry, dbLocation file.Location, resolver file.Resolver, release *linux.Release, evidence ...file.Location) pkg.Package {
// TODO: separate pr to license refactor, but explore extracting dpkg-specific license parsing into a separate function
licenses := make([]pkg.License, 0)

locations := file.NewLocationSet(dbLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
locations.Add(evidence...)

p := pkg.Package{
Name: d.Package,
Version: d.Version,
Licenses: pkg.NewLicenseSet(licenses...),
Locations: file.NewLocationSet(dbLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Locations: locations,
PURL: packageURL(d, release),
Type: pkg.DebPkg,
Metadata: d,
Expand Down Expand Up @@ -88,7 +92,7 @@ func packageURL(m pkg.DpkgDBEntry, distro *linux.Release) string {
func addLicenses(resolver file.Resolver, dbLocation file.Location, p *pkg.Package) {
metadata, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.WithFields("package", p).Warn("unable to extract DPKG metadata to add licenses")
log.WithFields("package", p).Trace("unable to extract DPKG metadata to add licenses")
return
}

Expand All @@ -110,7 +114,7 @@ func addLicenses(resolver file.Resolver, dbLocation file.Location, p *pkg.Packag
func mergeFileListing(resolver file.Resolver, dbLocation file.Location, p *pkg.Package) {
metadata, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.WithFields("package", p).Warn("unable to extract DPKG metadata to file listing")
log.WithFields("package", p).Trace("unable to extract DPKG metadata to file listing")
return
}

Expand Down Expand Up @@ -204,7 +208,7 @@ func fetchMd5Contents(resolver file.Resolver, dbLocation file.Location, m pkg.Dp
// this is unexpected, but not a show-stopper
md5Reader, err = resolver.FileContentsByLocation(*location)
if err != nil {
log.Warnf("failed to fetch deb md5 contents (package=%s): %+v", m.Package, err)
log.Tracef("failed to fetch deb md5 contents (package=%s): %+v", m.Package, err)
}

l := location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)
Expand Down Expand Up @@ -239,7 +243,7 @@ func fetchConffileContents(resolver file.Resolver, dbLocation file.Location, m p
// this is unexpected, but not a show-stopper
reader, err = resolver.FileContentsByLocation(*location)
if err != nil {
log.Warnf("failed to fetch deb conffiles contents (package=%s): %+v", m.Package, err)
log.Tracef("failed to fetch deb conffiles contents (package=%s): %+v", m.Package, err)
}

l := location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)
Expand All @@ -263,7 +267,7 @@ func fetchCopyrightContents(resolver file.Resolver, dbLocation file.Location, m

reader, err := resolver.FileContentsByLocation(*location)
if err != nil {
log.Warnf("failed to fetch deb copyright contents (package=%s): %s", m.Package, err)
log.Tracef("failed to fetch deb copyright contents (package=%s): %s", m.Package, err)
}
defer internal.CloseAndLogError(reader, location.RealPath)

Expand Down
33 changes: 32 additions & 1 deletion syft/pkg/cataloger/debian/parse_dpkg_db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"io"
"path"
"path/filepath"
"regexp"
"strings"

Expand Down Expand Up @@ -34,12 +36,41 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ

var pkgs []pkg.Package
for _, m := range metadata {
pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease))
p := newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease, findDpkgInfoFiles(m.Package, resolver, reader.Location)...)
pkgs = append(pkgs, p)
}

return pkgs, nil, nil
}

func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Location) []file.Location {
if resolver == nil {
return nil
}
if strings.TrimSpace(name) == "" {
return nil
}

// for typical debian-base distributions, the installed package info is at /var/lib/dpkg/status
// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
// the same directory
searchPath := filepath.Dir(dbLocation.RealPath)

if !strings.HasSuffix(searchPath, "status.d") {
searchPath = path.Join(searchPath, "info")
}

// look for /var/lib/dpkg/info/NAME.*
locations, err := resolver.FilesByGlob(path.Join(searchPath, name+".*"))
if err != nil {
log.WithFields("error", err, "pkg", name).Trace("failed to fetch related dpkg info files")
return nil
}

return locations
}

// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
func parseDpkgStatus(reader io.Reader) ([]pkg.DpkgDBEntry, error) {
buffedReader := bufio.NewReader(reader)
Expand Down
1 change: 1 addition & 0 deletions ...oger/debian/test-fixtures/image-distroless-deb/var/lib/dpkg/status.d/libsqlite3-0.preinst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# some shell script...
1 change: 1 addition & 0 deletions syft/pkg/cataloger/debian/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.preinst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# some shell script...
3 changes: 2 additions & 1 deletion syft/pkg/cataloger/ocaml/parse_opam_test.go
View file
Open in desktop
wagoodman marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ package ocaml
import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/stretchr/testify/assert"
)

func TestParseOpamPackage(t *testing.T) {
Expand Down
Loading