From 1e5efcfccf66b65892566c535e1ebcf6bce12684 Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Fri, 29 Apr 2022 17:36:45 +0200 Subject: [PATCH] sriov: add support as special PCI devices Add support to report SRIOV devices. Differently from GPU devices, we model SRIOV devices as special PCI devices, extending the `pci` package instead of introducing a new top-level package. This design emerged during the review of a previous proposal: https://github.com/jaypipes/ghw/pull/230/commits/9058f61f48a3c801e7817dd10905fc11e57c298c#r755312597 SRIOV devices are either Physical Functions or Virtual functions. The preferred representation for ghw is Physical Functions, whose dependent devices will be Virtual Functions; however, for the sake of practicality, the API also exposes soft references to Virtual Functions, so consumers of the API can access them directly and not navigating the parent devices. This patch also adds support in `ghwc`, to report the sriov information, and in the `snapshot` package, to make sure to capture all the files in sysfs that ghw cares about. Last but not least, lacking access to suitable non-linux systems, support is provided only on linux OS, even though the API tries hard not to be linux-specific. Resolves: https://github.com/jaypipes/ghw/issues/92 Signed-off-by: Francesco Romani --- README.md | 33 +++++++++ cmd/ghwc/commands/sriov.go | 35 +++++++++ pkg/pci/function.go | 35 +++++++++ pkg/pci/function_linux.go | 106 ++++++++++++++++++++++++++++ pkg/pci/function_stub.go | 13 ++++ pkg/pci/pci.go | 46 +++++++++++- pkg/pci/pci_linux.go | 3 +- pkg/pci/pci_stub.go | 2 +- pkg/snapshot/clonetree.go | 15 +++- pkg/snapshot/clonetree_pci_linux.go | 32 +++++++++ 10 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 cmd/ghwc/commands/sriov.go create mode 100644 pkg/pci/function.go create mode 100644 pkg/pci/function_linux.go create mode 100644 pkg/pci/function_stub.go diff --git a/README.md b/README.md index 4c64dca0..0748dc84 100644 --- a/README.md +++ b/README.md @@ -879,6 +879,39 @@ Subclass: VGA compatible controller [00] Programming Interface: VGA controller [00] ``` +#### SRIOV + +SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + pci, err := ghw.PCI() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info through PCI: %v", err) + } + + fmt.Printf("%v\n", pci) + + for _, dev := range pci.Functions { + fmt.Printf(" %v\n", dev) + } +} +``` + +`ghw` discovers the SRIOV devices by scanning PCI devices. Thus, you need to make sure to have scanned the PCI devices before +querying for SRIOV devices (aka "Functions", "PCI Functions"). +Virtual Functions (VFs) are hosted on Physical Functions (PFs). +Virtual Functions are available both as entries in the `pci.Functions` slice and as properties of their parent Physical Functions. +Both references are aliases to the same object. + ### GPU The `ghw.GPU()` function returns a `ghw.GPUInfo` struct that contains diff --git a/cmd/ghwc/commands/sriov.go b/cmd/ghwc/commands/sriov.go new file mode 100644 index 00000000..f03cd527 --- /dev/null +++ b/cmd/ghwc/commands/sriov.go @@ -0,0 +1,35 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package commands + +import ( + "github.com/jaypipes/ghw" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// sriovCmd represents the install command +var sriovCmd = &cobra.Command{ + Use: "sriov", + Short: "Show Single Root I/O Virtualization device information for the host system", + RunE: showSRIOV, +} + +// showSRIOV shows SRIOV information for the host system. +func showSRIOV(cmd *cobra.Command, args []string) error { + info, err := ghw.PCI() + if err != nil { + return errors.Wrap(err, "error getting SRIOV info through PCI") + } + + printInfo(info.DescribeDevices(info.GetSRIOVDevices())) + return nil +} + +func init() { + rootCmd.AddCommand(sriovCmd) +} diff --git a/pkg/pci/function.go b/pkg/pci/function.go new file mode 100644 index 00000000..cb33ba92 --- /dev/null +++ b/pkg/pci/function.go @@ -0,0 +1,35 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import "fmt" + +// Function describes an SR-IOV physical or virtual function. Physical functions +// will have no Parent Function struct pointer and will have one or more Function +// structs in the Virtual field. +type Function struct { + Parent *Function `json:"parent,omitempty"` + // MaxVirtual contains the maximum number of supported virtual + // functions for this physical function + MaxVirtual int `json:"max_virtual,omitempty"` + // Virtual contains the physical function's virtual functions + Virtual []*Function `json:"virtual_functions"` +} + +// IsPhysical returns true if the PCIe function is a physical function, false +// if it is a virtual function. It is safe to assume that if a function is not +// physical, then is virtual (e.g. can't be anything else) +func (f *Function) IsPhysical() bool { + return f.Parent == nil +} + +func (f *Function) String() string { + if f.IsPhysical() { + return fmt.Sprintf("function: 'physical' virtual: '%d/%d'", len(f.Virtual), f.MaxVirtual) + } + return "function: 'virtual'" +} diff --git a/pkg/pci/function_linux.go b/pkg/pci/function_linux.go new file mode 100644 index 00000000..0b84013a --- /dev/null +++ b/pkg/pci/function_linux.go @@ -0,0 +1,106 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/util" +) + +// GetSRIOVDevices returns only the PCI devices that are +// Single Root I/O Virtualization (SR-IOV) capable -- either +// physical of virtual functions. +func (i *Info) GetSRIOVDevices() []*Device { + res := []*Device{} + for _, dev := range i.Devices { + if dev.Function == nil { + continue + } + res = append(res, dev) + } + return res +} + +func (info *Info) fillSRIOVDevices() error { + for _, dev := range info.Devices { + isPF, err := info.fillPhysicalFunctionForDevice(dev) + if !isPF { + // not a physical function, nothing to do + continue + } + if err != nil { + return err + } + } + return nil +} + +func (info *Info) fillPhysicalFunctionForDevice(dev *Device) (bool, error) { + paths := linuxpath.New(info.ctx) + devPath := filepath.Join(paths.SysBusPciDevices, dev.Address) + + buf, err := ioutil.ReadFile(filepath.Join(devPath, "sriov_totalvfs")) + if err != nil { + // is not a physfn. Since we will fill virtfn from physfn, we can give up now + // note we intentionally swallow the error. + return false, nil + } + + maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf))) + if err != nil { + return true, fmt.Errorf("cannot reading sriov_totalvfn: %w", err) + } + + pf := &Function{ + MaxVirtual: maxVFs, + } + err = info.fillVirtualFunctionsForPhysicalFunction(pf, devPath) + if err != nil { + return true, fmt.Errorf("cannot inspect VFs: %w", err) + } + dev.Function = pf + return true, nil +} + +func (info *Info) fillVirtualFunctionsForPhysicalFunction(parentFn *Function, parentPath string) error { + numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs")) + if numVfs == -1 { + return fmt.Errorf("invalid number of virtual functions: %v", numVfs) + } + + var vfs []*Function + for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ { + virtFn := fmt.Sprintf("virtfn%d", vfnIdx) + vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn)) + if err != nil { + return fmt.Errorf("error reading backing device for virtfn %q: %w", virtFn, err) + } + + vfnAddr := filepath.Base(vfnDest) + vfnDev := info.GetDevice(vfnAddr) + if vfnDev == nil { + return fmt.Errorf("error finding the PCI device for virtfn %s", vfnAddr) + } + + // functions must be ordered by their index + vf := &Function{ + Parent: parentFn, + } + + vfs = append(vfs, vf) + vfnDev.Function = vf + } + + parentFn.Virtual = vfs + return nil +} diff --git a/pkg/pci/function_stub.go b/pkg/pci/function_stub.go new file mode 100644 index 00000000..b00997c0 --- /dev/null +++ b/pkg/pci/function_stub.go @@ -0,0 +1,13 @@ +//go:build !linux +// +build !linux + +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +func (info *Info) ListFunctions() []*Function { + return nil +} diff --git a/pkg/pci/pci.go b/pkg/pci/pci.go index 49adde62..77b62524 100644 --- a/pkg/pci/pci.go +++ b/pkg/pci/pci.go @@ -9,6 +9,7 @@ package pci import ( "encoding/json" "fmt" + "strings" "github.com/jaypipes/pcidb" @@ -36,6 +37,8 @@ type Device struct { // architecture is not NUMA. Node *topology.Node `json:"node,omitempty"` Driver string `json:"driver"` + // If this device is a SRIOV function, this field is non-nil + Function *Function `json:"function,omitempty"` } type devIdent struct { @@ -105,13 +108,18 @@ func (d *Device) String() string { if d.Class != nil { className = d.Class.Name } + fnInfo := "" + if d.Function != nil { + fnInfo = " " + d.Function.String() + } return fmt.Sprintf( - "%s -> driver: '%s' class: '%s' vendor: '%s' product: '%s'", + "%s -> driver: '%s' class: '%s' vendor: '%s' product: '%s'%s", d.Address, d.Driver, className, vendorName, productName, + fnInfo, ) } @@ -127,6 +135,42 @@ func (i *Info) String() string { return fmt.Sprintf("PCI (%d devices)", len(i.Devices)) } +type DevicesPrinter struct { + ctx *context.Context + devs []*Device +} + +func (dp DevicesPrinter) String() string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", dev.String()) + } + return sb.String() +} + +func (dp DevicesPrinter) JSONString(pretty bool) string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", marshal.SafeJSON(dp.ctx, dev, pretty)) + } + return sb.String() +} + +func (dp DevicesPrinter) YAMLString() string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", marshal.SafeYAML(dp.ctx, dev)) + } + return sb.String() +} + +func (i *Info) DescribeDevices(devs []*Device) DevicesPrinter { + return DevicesPrinter{ + ctx: i.ctx, + devs: devs, + } +} + // New returns a pointer to an Info struct that contains information about the // PCI devices on the host system func New(opts ...*option.Option) (*Info, error) { diff --git a/pkg/pci/pci_linux.go b/pkg/pci/pci_linux.go index 3c133ee9..f5398fcf 100644 --- a/pkg/pci/pci_linux.go +++ b/pkg/pci/pci_linux.go @@ -46,7 +46,8 @@ func (i *Info) load() error { } i.db = db i.Devices = i.getDevices() - return nil + // we need to do another pass once we filled all the PCI devices. + return i.fillSRIOVDevices() } func getDeviceModaliasPath(ctx *context.Context, pciAddr *pciaddr.Address) string { diff --git a/pkg/pci/pci_stub.go b/pkg/pci/pci_stub.go index 9ebb396d..9e8b345b 100644 --- a/pkg/pci/pci_stub.go +++ b/pkg/pci/pci_stub.go @@ -15,7 +15,7 @@ import ( ) func (i *Info) load() error { - return errors.New("pciFillInfo not implemented on " + runtime.GOOS) + return errors.New("pci load() not implemented on " + runtime.GOOS) } // GetDevice returns a pointer to a Device struct that describes the PCI diff --git a/pkg/snapshot/clonetree.go b/pkg/snapshot/clonetree.go index 519a874d..587c6b03 100644 --- a/pkg/snapshot/clonetree.go +++ b/pkg/snapshot/clonetree.go @@ -100,7 +100,7 @@ func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) er if opts == nil { opts = &CopyFileOptions{ IsSymlinkFn: isSymlink, - ShouldCreateDirFn: isDriversDir, + ShouldCreateDirFn: shouldCreateDir, } } for _, fileSpec := range fileSpecs { @@ -157,6 +157,13 @@ func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) err return nil } +func shouldCreateDir(path string, fi os.FileInfo) bool { + if isDeviceNetworkDir(path, fi) { + return true + } + return isDriversDir(path, fi) +} + func isSymlink(path string, fi os.FileInfo) bool { return fi.Mode()&os.ModeSymlink != 0 } @@ -165,6 +172,12 @@ func isDriversDir(path string, fi os.FileInfo) bool { return strings.Contains(path, "drivers") } +func isDeviceNetworkDir(path string, fi os.FileInfo) bool { + parentDir := filepath.Base(filepath.Dir(path)) + // TODO: the "HasPrefix" check is brutal, but should work on linux + return parentDir == "net" && strings.HasPrefix(path, "/sys/devices") +} + func copyLink(path, targetPath string) error { target, err := os.Readlink(path) if err != nil { diff --git a/pkg/snapshot/clonetree_pci_linux.go b/pkg/snapshot/clonetree_pci_linux.go index dbc3fc83..e39e8c09 100644 --- a/pkg/snapshot/clonetree_pci_linux.go +++ b/pkg/snapshot/clonetree_pci_linux.go @@ -70,6 +70,19 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { "revision", "vendor", } + + perDevEntriesOpt := []string{ + "driver", + "net/*", + "physfn", + "sriov_*", + "virtfn*", + } + + ignoreSet := map[string]bool{ + "sriov_vf_msix_count": true, // linux >= 5.14, write-only + } + entries, err := ioutil.ReadDir(root) if err != nil { return []string{}, []string{} @@ -96,6 +109,25 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) } + for _, perNetEntryOpt := range perDevEntriesOpt { + netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt) + + items, err := filepath.Glob(netEntryOptPath) + if err != nil { + // TODO: we skip silently because we don't have + // a ctx handy, so we can't do ctx.Warn :\ + continue + } + + for _, item := range items { + globbedEntry := filepath.Base(item) + if _, ok := ignoreSet[globbedEntry]; ok { + continue + } + fileSpecs = append(fileSpecs, item) + } + } + if isPCIBridge(entryPath) { trace("adding new PCI root %q\n", entryName) pciRoots = append(pciRoots, pciEntry)