diff --git a/README.md b/README.md index dd534b0f..96ce2ec6 100644 --- a/README.md +++ b/README.md @@ -1115,6 +1115,67 @@ information `ghw.TopologyNode` struct if you'd like to dig deeper into the NUMA/topology subsystem +### SRIOV + +*This API is PROVISIONAL! ghw will try hard to not make breaking changes to this API, but still users are advice this new API is not +declared stable yet. We expect to declare it stable with ghw version 1.0.0* + +SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly, like gpus. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + sriov, err := ghw.SRIOV() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info: %v", err) + } + + fmt.Printf("%v\n", sriov) + + for _, dev := range sriov.PhysicalFunctions { + fmt.Printf(" %v\n", dev) + } +} +``` + +ghw discovers the SRIOV devices starting from the Physical Function (PF) and exposes them in the `PhysicalFunctions` slice. +Virtual Function (VF) are exposed as properties of the PF instance with exposes them. +However, in some cases users are interested in the VFs first, so it's clumsy to navigate the PFs to learn about VFs. +To make this easier, ghw also exposes a slice of VF instances. These instances are soft references to the very same VF objects +you can find from the PF objects. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + sriov, err := ghw.SRIOV() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info: %v", err) + } + + fmt.Printf("%v\n", sriov) + + // you will see the very same VF data data you seen from the previous example + for _, dev := range sriov.VirtualFunctions { + fmt.Printf(" %v\n", dev) + } +} +``` + + ### Chassis The host's chassis information is accessible with the `ghw.Chassis()` function. This diff --git a/alias.go b/alias.go index 666200bc..cfadf597 100644 --- a/alias.go +++ b/alias.go @@ -19,6 +19,7 @@ import ( "github.com/jaypipes/ghw/pkg/pci" pciaddress "github.com/jaypipes/ghw/pkg/pci/address" "github.com/jaypipes/ghw/pkg/product" + "github.com/jaypipes/ghw/pkg/sriov" "github.com/jaypipes/ghw/pkg/topology" ) @@ -150,3 +151,11 @@ type GraphicsCard = gpu.GraphicsCard var ( GPU = gpu.New ) + +type SRIOVInfo = sriov.Info +type PhysicalFunction = sriov.PhysicalFunction +type VirtualFunction = sriov.VirtualFunction + +var ( + SRIOV = sriov.New +) diff --git a/cmd/ghwc/commands/sriov.go b/cmd/ghwc/commands/sriov.go new file mode 100644 index 00000000..c6048c8d --- /dev/null +++ b/cmd/ghwc/commands/sriov.go @@ -0,0 +1,48 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package commands + +import ( + "fmt" + + "github.com/jaypipes/ghw" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// sriovCmd represents the listing command +var sriovCmd = &cobra.Command{ + Use: "sriov", + Short: "Show SRIOV devices information for the host system", + RunE: showSRIOV, +} + +// showSRIOV show SRIOV physical device information for the host system. +func showSRIOV(cmd *cobra.Command, args []string) error { + sriov, err := ghw.SRIOV() + if err != nil { + return errors.Wrap(err, "error getting SRIOV info") + } + + switch outputFormat { + case outputFormatHuman: + fmt.Printf("%v\n", sriov) + + for _, dev := range sriov.PhysicalFunctions { + fmt.Printf(" %v\n", dev) + } + case outputFormatJSON: + fmt.Printf("%s\n", sriov.JSONString(pretty)) + case outputFormatYAML: + fmt.Printf("%s", sriov.YAMLString()) + } + return nil +} + +func init() { + rootCmd.AddCommand(sriovCmd) +} diff --git a/host.go b/host.go index 5d82a53a..a9905011 100644 --- a/host.go +++ b/host.go @@ -8,6 +8,7 @@ package ghw import ( "fmt" + "strings" "github.com/jaypipes/ghw/pkg/context" @@ -22,6 +23,7 @@ import ( "github.com/jaypipes/ghw/pkg/net" "github.com/jaypipes/ghw/pkg/pci" "github.com/jaypipes/ghw/pkg/product" + "github.com/jaypipes/ghw/pkg/sriov" "github.com/jaypipes/ghw/pkg/topology" ) @@ -40,6 +42,7 @@ type HostInfo struct { Baseboard *baseboard.Info `json:"baseboard"` Product *product.Info `json:"product"` PCI *pci.Info `json:"pci"` + SRIOV *sriov.Info `json:"sriov"` } // Host returns a pointer to a HostInfo struct that contains fields with @@ -91,6 +94,10 @@ func Host(opts ...*WithOption) (*HostInfo, error) { if err != nil { return nil, err } + sriovInfo, err := sriov.New(opts...) + if err != nil { + return nil, err + } return &HostInfo{ ctx: ctx, CPU: cpuInfo, @@ -104,26 +111,32 @@ func Host(opts ...*WithOption) (*HostInfo, error) { Baseboard: baseboardInfo, Product: productInfo, PCI: pciInfo, + SRIOV: sriovInfo, }, nil } // String returns a newline-separated output of the HostInfo's component // structs' String-ified output func (info *HostInfo) String() string { - return fmt.Sprintf( - "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", - info.Block.String(), - info.CPU.String(), - info.GPU.String(), - info.Memory.String(), - info.Network.String(), - info.Topology.String(), - info.Chassis.String(), - info.BIOS.String(), - info.Baseboard.String(), - info.Product.String(), - info.PCI.String(), - ) + var b strings.Builder + for _, s := range []fmt.Stringer{ + info.Block, + info.CPU, + info.GPU, + info.Memory, + info.Network, + info.Topology, + info.Chassis, + info.BIOS, + info.Baseboard, + info.Product, + info.PCI, + info.SRIOV, + } { + b.WriteString(s.String()) + b.WriteString("\n") + } + return b.String() } // YAMLString returns a string with the host information formatted as YAML diff --git a/pkg/pci/address/address.go b/pkg/pci/address/address.go index 6a8a4e45..ad371055 100644 --- a/pkg/pci/address/address.go +++ b/pkg/pci/address/address.go @@ -25,6 +25,16 @@ type Address struct { Function string } +func (addr *Address) Equal(a *Address) bool { + if addr == nil && a == nil { + return true + } + if addr != nil && a != nil { + return addr.Domain == a.Domain && addr.Bus == a.Bus && addr.Device == a.Device && addr.Function == a.Function + } + return false +} + // String() returns the canonical [D]BDF representation of this Address func (addr *Address) String() string { return addr.Domain + ":" + addr.Bus + ":" + addr.Device + "." + addr.Function diff --git a/pkg/pci/address/address_test.go b/pkg/pci/address/address_test.go index daf0eaa6..c3989dd4 100644 --- a/pkg/pci/address/address_test.go +++ b/pkg/pci/address/address_test.go @@ -80,3 +80,11 @@ func TestPCIAddressFromString(t *testing.T) { } } } + +func TestPCIAddressEqual(t *testing.T) { + addr1 := pciaddr.FromString("0000:03:00.A") + addr2 := pciaddr.FromString("03:00.A") + if addr1.Equal(addr2) == false { + t.Fatalf("addr1 %v and addr2 %v should be equal", addr1, addr2) + } +} diff --git a/pkg/snapshot/clonetree_pci_linux.go b/pkg/snapshot/clonetree_pci_linux.go index dbc3fc83..5f98e903 100644 --- a/pkg/snapshot/clonetree_pci_linux.go +++ b/pkg/snapshot/clonetree_pci_linux.go @@ -70,6 +70,15 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { "revision", "vendor", } + + perDevEntriesOpt := []string{ + "driver", + "net/*", + "physfn", + "sriov_*", + "virtfn*", + } + entries, err := ioutil.ReadDir(root) if err != nil { return []string{}, []string{} @@ -96,6 +105,14 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) } + for _, perNetEntryOpt := range perDevEntriesOpt { + netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt) + if items, err := filepath.Glob(netEntryOptPath); err == nil && len(items) > 0 { + fileSpecs = append(fileSpecs, netEntryOptPath) + } + + } + if isPCIBridge(entryPath) { trace("adding new PCI root %q\n", entryName) pciRoots = append(pciRoots, pciEntry) diff --git a/pkg/sriov/sriov.go b/pkg/sriov/sriov.go new file mode 100644 index 00000000..f29c6d8f --- /dev/null +++ b/pkg/sriov/sriov.go @@ -0,0 +1,113 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov + +import ( + "fmt" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/marshal" + "github.com/jaypipes/ghw/pkg/option" + "github.com/jaypipes/ghw/pkg/pci" + pciaddr "github.com/jaypipes/ghw/pkg/pci/address" +) + +type Device struct { + Interfaces []string `json:"interfaces"` + // the PCI address where the SRIOV instance can be found + Address *pciaddr.Address `json:"address"` + PCI *pci.Device `json:"pci"` +} + +func (d Device) ToString(devType string) string { + deviceStr := d.Address.String() + nodeStr := "" + if d.PCI != nil { + deviceStr = d.PCI.String() + if d.PCI.Node != nil { + nodeStr = fmt.Sprintf(" [affined to NUMA node %d]", d.PCI.Node.ID) + } + } + return fmt.Sprintf("%s function %s@%s", devType, nodeStr, deviceStr) +} + +type PhysicalFunction struct { + Device + MaxVFNum int `json:"max_vf_num,omitempty"` + VFs []VirtualFunction `json:"vfs,omitempty"` +} + +type VirtualFunction struct { + Device + ID int `json:"id"` + // Address of the (parent) Physical Function this Virtual Function pertains to. + ParentAddress *pciaddr.Address `json:"parent_address,omitempty"` +} + +func (pf *PhysicalFunction) String() string { + return fmt.Sprintf("%s with %d/%d virtual functions", + pf.Device.ToString("physical"), + len(pf.VFs), + pf.MaxVFNum, + ) +} + +func (vf *VirtualFunction) String() string { + return fmt.Sprintf("%s index %d from %s", + vf.Device.ToString("virtual"), + vf.ID, + vf.ParentAddress, + ) +} + +type Info struct { + ctx *context.Context + // All the Physical Functions found in the host system, + PhysicalFunctions []*PhysicalFunction `json:"physical_functions,omitempty"` + // All the Virtual Functions found in the host system, + // This is the very same data found navigating the `PhysicalFunctions`; + // These pointers point back to the corresponding structs in the `PhysicalFunctions` + // slice. + VirtualFunctions []*VirtualFunction `json:"virtual_functions,omitempty"` +} + +// New returns a pointer to an Info struct that contains information about the +// SRIOV devices on the host system. +func New(opts ...*option.Option) (*Info, error) { + ctx := context.New(opts...) + info := &Info{ctx: ctx} + if err := ctx.Do(info.load); err != nil { + return nil, err + } + return info, nil +} + +func (i *Info) String() string { + return fmt.Sprintf( + "sriov (%d phsyical %d virtual devices)", + len(i.PhysicalFunctions), + len(i.VirtualFunctions), + ) +} + +// simple private struct used to encapsulate SRIOV information in a top-level +// "sriov" YAML/JSON map/object key +type sriovPrinter struct { + Info *Info `json:"sriov,omitempty"` +} + +// YAMLString returns a string with the SRIOV information formatted as YAML +// under a top-level "sriov:" key +func (i *Info) YAMLString() string { + return marshal.SafeYAML(i.ctx, sriovPrinter{i}) +} + +// JSONString returns a string with the SRIOV information formatted as JSON +// under a top-level "sriov:" key +func (i *Info) JSONString(indent bool) string { + return marshal.SafeJSON(i.ctx, sriovPrinter{i}, indent) +} diff --git a/pkg/sriov/sriov_linux.go b/pkg/sriov/sriov_linux.go new file mode 100644 index 00000000..0d03596f --- /dev/null +++ b/pkg/sriov/sriov_linux.go @@ -0,0 +1,128 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/pci" + pciaddress "github.com/jaypipes/ghw/pkg/pci/address" + "github.com/jaypipes/ghw/pkg/util" +) + +func (info *Info) load() error { + // SRIOV device do not have a specific class (as in "entry in /sys/class"), + // so we need to iterate over all the PCI devices. + pciInfo, err := pci.New(context.WithContext(info.ctx)) + if err != nil { + return err + } + + for _, dev := range pciInfo.Devices { + err := info.scanDevice(pciInfo, dev) + if err != nil { + return err + } + } + + return nil +} + +func (info *Info) scanDevice(pciInfo *pci.Info, dev *pci.Device) error { + paths := linuxpath.New(info.ctx) + devPath := filepath.Join(paths.SysBusPciDevices, dev.Address) + + buf, err := ioutil.ReadFile(filepath.Join(devPath, "sriov_totalvfs")) + if err != nil { + // is not a physfn. Since we will fill virtfn from physfn, we can give up now + return nil + } + + maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf))) + if err != nil { + info.ctx.Warn("error reading sriov_totalvfn for %q: %v", devPath, err) + return nil + } + + virtFNs := findVFsFromPF(info, pciInfo, dev.Address, devPath) + physFN := PhysicalFunction{ + Device: info.newDevice(dev, devPath), + MaxVFNum: maxVFs, + VFs: virtFNs, + } + + info.PhysicalFunctions = append(info.PhysicalFunctions, &physFN) + for idx := 0; idx < len(virtFNs); idx++ { + info.VirtualFunctions = append(info.VirtualFunctions, &virtFNs[idx]) + } + + return nil +} + +func findVFsFromPF(info *Info, pciInfo *pci.Info, parentAddr, parentPath string) []VirtualFunction { + numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs")) + if numVfs == -1 { + return nil + } + + var vfs []VirtualFunction + for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ { + virtFn := fmt.Sprintf("virtfn%d", vfnIdx) + vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn)) + if err != nil { + info.ctx.Warn("error reading backing device for virtfn %q physfn %q: %v", virtFn, parentPath, err) + return nil + } + + vfnPath := filepath.Clean(filepath.Join(parentPath, vfnDest)) + vfnAddr := filepath.Base(vfnDest) + vfnDev := pciInfo.GetDevice(vfnAddr) + if vfnDev == nil { + info.ctx.Warn("error finding the PCI device for virtfn %s physfn %s", vfnAddr, parentAddr) + return nil + } + + vfs = append(vfs, VirtualFunction{ + Device: info.newDevice(vfnDev, vfnPath), + ID: vfnIdx, + ParentAddress: pciaddress.FromString(parentAddr), + }) + } + return vfs +} + +func (info *Info) newDevice(dev *pci.Device, devPath string) Device { + // see: https://doc.dpdk.org/guides/linux_gsg/linux_drivers.html + return Device{ + Address: pciaddress.FromString(dev.Address), + Interfaces: findNetworks(info.ctx, devPath), + PCI: dev, + } +} + +func findNetworks(ctx *context.Context, devPath string) []string { + netPath := filepath.Join(devPath, "net") + + netEntries, err := ioutil.ReadDir(netPath) + if err != nil { + ctx.Warn("cannot enumerate network names for %q: %v", devPath, err) + return nil + } + + var networks []string + for _, netEntry := range netEntries { + networks = append(networks, netEntry.Name()) + } + + return networks +} diff --git a/pkg/sriov/sriov_linux_test.go b/pkg/sriov/sriov_linux_test.go new file mode 100644 index 00000000..49f90131 --- /dev/null +++ b/pkg/sriov/sriov_linux_test.go @@ -0,0 +1,198 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jaypipes/ghw/pkg/option" + pciaddr "github.com/jaypipes/ghw/pkg/pci/address" + "github.com/jaypipes/ghw/pkg/sriov" + + "github.com/jaypipes/ghw/testdata" +) + +// nolint: gocyclo +func TestStringify(t *testing.T) { + info := sriovTestSetup(t) + + for _, physFn := range info.PhysicalFunctions { + s := physFn.String() + if s == "" || !strings.Contains(s, "function") || !strings.Contains(s, "physical") { + t.Errorf("Wrong string representation %q", s) + } + } + + for _, virtFn := range info.VirtualFunctions { + s := virtFn.String() + if s == "" || !strings.Contains(s, "function") || !strings.Contains(s, "virtual") { + t.Errorf("Wrong string representation %q", s) + } + } + +} + +// nolint: gocyclo +func TestCountDevices(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + expectedPhysDevs := 2 + expectedVirtDevsPerPhys := 4 + numPhysDevs := len(info.PhysicalFunctions) + if numPhysDevs != expectedPhysDevs { + t.Errorf("Expected %d physical devices found %d", expectedPhysDevs, numPhysDevs) + } + numVirtDevs := len(info.VirtualFunctions) + if numPhysDevs*expectedVirtDevsPerPhys != numVirtDevs { + t.Errorf("Expected %d=(%d*%d) virtual devices found %d", numPhysDevs*expectedVirtDevsPerPhys, numPhysDevs, expectedVirtDevsPerPhys, numVirtDevs) + } + + for _, physDev := range info.PhysicalFunctions { + numVFs := len(physDev.VFs) + if numVFs != expectedVirtDevsPerPhys { + t.Errorf("Expected %d virtual devices for PF %s found %d", expectedVirtDevsPerPhys, physDev.Address.String(), numVFs) + } + } +} + +type pfTestCase struct { + addr string + netname string +} + +// nolint: gocyclo +func TestMatchPhysicalFunction(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + for _, pfTC := range []pfTestCase{ + { + addr: "0000:05:00.0", + netname: "enp5s0f0", + }, + { + addr: "0000:05:00.1", + netname: "enp5s0f1", + }, + } { + addr := pciaddr.FromString(pfTC.addr) + pf := findPF(info.PhysicalFunctions, addr) + if pf == nil { + t.Fatalf("missing PF at addr %q", addr.String()) + } + if pf.PCI == nil { + t.Errorf("missing PCI device for %q", addr.String()) + } + if pf.PCI.Driver != "igb" { + t.Errorf("unexpected driver for %#v: %q", pf, pf.PCI.Driver) + } + if len(pf.Interfaces) != 1 || pf.Interfaces[0] != pfTC.netname { + t.Errorf("unexpected interfaces for %#v: %v", pf, pf.Interfaces) + } + if pf.MaxVFNum != 7 { + t.Errorf("unexpected MaxVFNum for %#v: %d", pf, pf.MaxVFNum) + } + if len(pf.VFs) != 4 { + t.Errorf("unexpected VF count for %#v: %d", pf, len(pf.VFs)) + } + for _, vfInst := range pf.VFs { + vf := findVF(info.VirtualFunctions, vfInst.Address) + if vf == nil { + t.Errorf("VF %#v from %#v not found among info.VirtualFunctions", vfInst, pf) + } + } + } +} + +func TestMatchVirtualFunction(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + + for _, vf := range info.VirtualFunctions { + if vf.PCI == nil { + t.Errorf("missing PCI device for %q", vf.Address.String()) + } + if vf.PCI.Driver != "igbvf" { + t.Errorf("unexpected driver for %#v: %q", vf, vf.PCI.Driver) + } + + pf := findPF(info.PhysicalFunctions, vf.ParentAddress) + if pf == nil { + t.Fatalf("missing parent device for %q", vf.Address.String()) + } + if vf2 := findVFInst(pf.VFs, vf.Address); vf2 == nil { + t.Errorf("VF %#v not included in parent %#v VFs", vf, pf) + } + } +} + +func findPF(pfs []*sriov.PhysicalFunction, addr *pciaddr.Address) *sriov.PhysicalFunction { + for _, pf := range pfs { + if pf.Address.Equal(addr) { + return pf + } + } + return nil +} + +func findVF(vfs []*sriov.VirtualFunction, addr *pciaddr.Address) *sriov.VirtualFunction { + for _, vf := range vfs { + if vf.Address.Equal(addr) { + return vf + } + } + return nil +} + +func findVFInst(vfs []sriov.VirtualFunction, addr *pciaddr.Address) *sriov.VirtualFunction { + for idx := 0; idx < len(vfs); idx++ { + if vfs[idx].Address.Equal(addr) { + return &vfs[idx] + } + } + return nil +} + +func sriovTestSetup(t *testing.T) *sriov.Info { + if _, ok := os.LookupEnv("GHW_TESTING_SKIP_SRIOV"); ok { + t.Skip("Skipping SRIOV tests.") + } + + testdataPath, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatalf("Expected nil err, but got %v", err) + } + + multiNumaSnapshot := filepath.Join(testdataPath, "linux-amd64-intel-xeon-L5640.tar.gz") + // from now on we use constants reflecting the content of the snapshot we requested, + // which we reviewed beforehand. IOW, you need to know the content of the + // snapshot to fully understand this test. Inspect it using + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + + info, err := sriov.New(option.WithSnapshot(option.SnapshotOptions{ + Path: multiNumaSnapshot, + })) + + if err != nil { + t.Fatalf("Expected nil err, but got %v", err) + } + if info == nil { + t.Fatalf("Expected non-nil SRIOVInfo, but got nil") + } + return info +} diff --git a/pkg/sriov/sriov_stub.go b/pkg/sriov/sriov_stub.go new file mode 100644 index 00000000..55b8e708 --- /dev/null +++ b/pkg/sriov/sriov_stub.go @@ -0,0 +1,17 @@ +// +build !linux +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov + +import ( + "runtime" + + "github.com/pkg/errors" +) + +func (i *Info) load() error { + return errors.New("SRIOV load() not implemented on " + runtime.GOOS) +}