diff --git a/README.md b/README.md index 061b89cf..9beee02e 100644 --- a/README.md +++ b/README.md @@ -879,6 +879,39 @@ Subclass: VGA compatible controller [00] Programming Interface: VGA controller [00] ``` +#### SRIOV + +SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + pci, err := ghw.PCI() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info through PCI: %v", err) + } + + fmt.Printf("%v\n", pci) + + for _, dev := range pci.GetSRIOVDevices() { + fmt.Printf(" %v\n", dev) + } +} +``` + +`ghw` discovers the SRIOV devices by scanning PCI devices. Thus, you need to make sure to have scanned the PCI devices before +querying for SRIOV devices (aka "Functions", "PCI Functions"). +Virtual Functions (VFs) are hosted on Physical Functions (PFs). +Virtual Functions are available both as entries in the `pci.Functions` slice and as properties of their parent Physical Functions. +Both references are aliases to the same object. + ### GPU The `ghw.GPU()` function returns a `ghw.GPUInfo` struct that contains diff --git a/cmd/ghwc/commands/sriov.go b/cmd/ghwc/commands/sriov.go new file mode 100644 index 00000000..f03cd527 --- /dev/null +++ b/cmd/ghwc/commands/sriov.go @@ -0,0 +1,35 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package commands + +import ( + "github.com/jaypipes/ghw" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// sriovCmd represents the install command +var sriovCmd = &cobra.Command{ + Use: "sriov", + Short: "Show Single Root I/O Virtualization device information for the host system", + RunE: showSRIOV, +} + +// showSRIOV shows SRIOV information for the host system. +func showSRIOV(cmd *cobra.Command, args []string) error { + info, err := ghw.PCI() + if err != nil { + return errors.Wrap(err, "error getting SRIOV info through PCI") + } + + printInfo(info.DescribeDevices(info.GetSRIOVDevices())) + return nil +} + +func init() { + rootCmd.AddCommand(sriovCmd) +} diff --git a/pkg/pci/function.go b/pkg/pci/function.go new file mode 100644 index 00000000..cb33ba92 --- /dev/null +++ b/pkg/pci/function.go @@ -0,0 +1,35 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import "fmt" + +// Function describes an SR-IOV physical or virtual function. Physical functions +// will have no Parent Function struct pointer and will have one or more Function +// structs in the Virtual field. +type Function struct { + Parent *Function `json:"parent,omitempty"` + // MaxVirtual contains the maximum number of supported virtual + // functions for this physical function + MaxVirtual int `json:"max_virtual,omitempty"` + // Virtual contains the physical function's virtual functions + Virtual []*Function `json:"virtual_functions"` +} + +// IsPhysical returns true if the PCIe function is a physical function, false +// if it is a virtual function. It is safe to assume that if a function is not +// physical, then is virtual (e.g. can't be anything else) +func (f *Function) IsPhysical() bool { + return f.Parent == nil +} + +func (f *Function) String() string { + if f.IsPhysical() { + return fmt.Sprintf("function: 'physical' virtual: '%d/%d'", len(f.Virtual), f.MaxVirtual) + } + return "function: 'virtual'" +} diff --git a/pkg/pci/function_linux.go b/pkg/pci/function_linux.go new file mode 100644 index 00000000..262b87d1 --- /dev/null +++ b/pkg/pci/function_linux.go @@ -0,0 +1,105 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/util" +) + +// GetSRIOVDevices returns only the PCI devices that are +// Single Root I/O Virtualization (SR-IOV) capable -- either +// physical of virtual functions. +func (i *Info) GetSRIOVDevices() []*Device { + res := []*Device{} + for _, dev := range i.Devices { + if dev.Function == nil { + continue + } + res = append(res, dev) + } + return res +} + +func (info *Info) fillSRIOVDevices() error { + for _, dev := range info.Devices { + isPF, err := info.fillPhysicalFunctionForDevice(dev) + if !isPF { + // not a physical function, nothing to do + continue + } + if err != nil { + return err + } + } + return nil +} + +func (info *Info) fillPhysicalFunctionForDevice(dev *Device) (bool, error) { + paths := linuxpath.New(info.ctx) + devPath := filepath.Join(paths.SysBusPciDevices, dev.Address) + + buf, err := os.ReadFile(filepath.Join(devPath, "sriov_totalvfs")) + if err != nil { + // is not a physfn. Since we will fill virtfn from physfn, we can give up now + // note we intentionally swallow the error. + return false, nil + } + + maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf))) + if err != nil { + return true, fmt.Errorf("cannot reading sriov_totalvfn: %w", err) + } + + pf := &Function{ + MaxVirtual: maxVFs, + } + err = info.fillVirtualFunctionsForPhysicalFunction(pf, devPath) + if err != nil { + return true, fmt.Errorf("cannot inspect VFs: %w", err) + } + dev.Function = pf + return true, nil +} + +func (info *Info) fillVirtualFunctionsForPhysicalFunction(parentFn *Function, parentPath string) error { + numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs")) + if numVfs == -1 { + return fmt.Errorf("invalid number of virtual functions: %v", numVfs) + } + + var vfs []*Function + for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ { + virtFn := fmt.Sprintf("virtfn%d", vfnIdx) + vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn)) + if err != nil { + return fmt.Errorf("error reading backing device for virtfn %q: %w", virtFn, err) + } + + vfnAddr := filepath.Base(vfnDest) + vfnDev := info.GetDevice(vfnAddr) + if vfnDev == nil { + return fmt.Errorf("error finding the PCI device for virtfn %s", vfnAddr) + } + + // functions must be ordered by their index + vf := &Function{ + Parent: parentFn, + } + + vfs = append(vfs, vf) + vfnDev.Function = vf + } + + parentFn.Virtual = vfs + return nil +} diff --git a/pkg/pci/function_stub.go b/pkg/pci/function_stub.go new file mode 100644 index 00000000..66ef284d --- /dev/null +++ b/pkg/pci/function_stub.go @@ -0,0 +1,13 @@ +//go:build !linux +// +build !linux + +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +func (i *Info) GetSRIOVDevices() []*Device { + return nil +} diff --git a/pkg/pci/pci.go b/pkg/pci/pci.go index 49adde62..77b62524 100644 --- a/pkg/pci/pci.go +++ b/pkg/pci/pci.go @@ -9,6 +9,7 @@ package pci import ( "encoding/json" "fmt" + "strings" "github.com/jaypipes/pcidb" @@ -36,6 +37,8 @@ type Device struct { // architecture is not NUMA. Node *topology.Node `json:"node,omitempty"` Driver string `json:"driver"` + // If this device is a SRIOV function, this field is non-nil + Function *Function `json:"function,omitempty"` } type devIdent struct { @@ -105,13 +108,18 @@ func (d *Device) String() string { if d.Class != nil { className = d.Class.Name } + fnInfo := "" + if d.Function != nil { + fnInfo = " " + d.Function.String() + } return fmt.Sprintf( - "%s -> driver: '%s' class: '%s' vendor: '%s' product: '%s'", + "%s -> driver: '%s' class: '%s' vendor: '%s' product: '%s'%s", d.Address, d.Driver, className, vendorName, productName, + fnInfo, ) } @@ -127,6 +135,42 @@ func (i *Info) String() string { return fmt.Sprintf("PCI (%d devices)", len(i.Devices)) } +type DevicesPrinter struct { + ctx *context.Context + devs []*Device +} + +func (dp DevicesPrinter) String() string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", dev.String()) + } + return sb.String() +} + +func (dp DevicesPrinter) JSONString(pretty bool) string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", marshal.SafeJSON(dp.ctx, dev, pretty)) + } + return sb.String() +} + +func (dp DevicesPrinter) YAMLString() string { + var sb strings.Builder + for _, dev := range dp.devs { + fmt.Fprintf(&sb, "%s\n", marshal.SafeYAML(dp.ctx, dev)) + } + return sb.String() +} + +func (i *Info) DescribeDevices(devs []*Device) DevicesPrinter { + return DevicesPrinter{ + ctx: i.ctx, + devs: devs, + } +} + // New returns a pointer to an Info struct that contains information about the // PCI devices on the host system func New(opts ...*option.Option) (*Info, error) { diff --git a/pkg/pci/pci_linux.go b/pkg/pci/pci_linux.go index da9857d4..4377cbf7 100644 --- a/pkg/pci/pci_linux.go +++ b/pkg/pci/pci_linux.go @@ -45,7 +45,8 @@ func (i *Info) load() error { } i.db = db i.Devices = i.getDevices() - return nil + // we need to do another pass once we filled all the PCI devices. + return i.fillSRIOVDevices() } func getDeviceModaliasPath(ctx *context.Context, pciAddr *pciaddr.Address) string { diff --git a/pkg/pci/pci_stub.go b/pkg/pci/pci_stub.go index 9ebb396d..9e8b345b 100644 --- a/pkg/pci/pci_stub.go +++ b/pkg/pci/pci_stub.go @@ -15,7 +15,7 @@ import ( ) func (i *Info) load() error { - return errors.New("pciFillInfo not implemented on " + runtime.GOOS) + return errors.New("pci load() not implemented on " + runtime.GOOS) } // GetDevice returns a pointer to a Device struct that describes the PCI diff --git a/pkg/snapshot/clonetree.go b/pkg/snapshot/clonetree.go index 020e7e67..6378491d 100644 --- a/pkg/snapshot/clonetree.go +++ b/pkg/snapshot/clonetree.go @@ -99,7 +99,7 @@ func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) er if opts == nil { opts = &CopyFileOptions{ IsSymlinkFn: isSymlink, - ShouldCreateDirFn: isDriversDir, + ShouldCreateDirFn: shouldCreateDir, } } for _, fileSpec := range fileSpecs { @@ -156,6 +156,13 @@ func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) err return nil } +func shouldCreateDir(path string, fi os.FileInfo) bool { + if isDeviceNetworkDir(path, fi) { + return true + } + return isDriversDir(path, fi) +} + func isSymlink(path string, fi os.FileInfo) bool { return fi.Mode()&os.ModeSymlink != 0 } @@ -164,6 +171,12 @@ func isDriversDir(path string, fi os.FileInfo) bool { return strings.Contains(path, "drivers") } +func isDeviceNetworkDir(path string, fi os.FileInfo) bool { + parentDir := filepath.Base(filepath.Dir(path)) + // TODO: the "HasPrefix" check is brutal, but should work on linux + return parentDir == "net" && strings.HasPrefix(path, "/sys/devices") +} + func copyLink(path, targetPath string) error { target, err := os.Readlink(path) if err != nil { diff --git a/pkg/snapshot/clonetree_pci_linux.go b/pkg/snapshot/clonetree_pci_linux.go index e7aa7d26..786161ff 100644 --- a/pkg/snapshot/clonetree_pci_linux.go +++ b/pkg/snapshot/clonetree_pci_linux.go @@ -69,6 +69,19 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { "revision", "vendor", } + + perDevEntriesOpt := []string{ + "driver", + "net/*", + "physfn", + "sriov_*", + "virtfn*", + } + + ignoreSet := map[string]bool{ + "sriov_vf_msix_count": true, // linux >= 5.14, write-only + } + entries, err := os.ReadDir(root) if err != nil { return []string{}, []string{} @@ -95,6 +108,25 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) } + for _, perNetEntryOpt := range perDevEntriesOpt { + netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt) + + items, err := filepath.Glob(netEntryOptPath) + if err != nil { + // TODO: we skip silently because we don't have + // a ctx handy, so we can't do ctx.Warn :\ + continue + } + + for _, item := range items { + globbedEntry := filepath.Base(item) + if _, ok := ignoreSet[globbedEntry]; ok { + continue + } + fileSpecs = append(fileSpecs, item) + } + } + if isPCIBridge(entryPath) { trace("adding new PCI root %q\n", entryName) pciRoots = append(pciRoots, pciEntry)