From 492ce21ad70fa5f14ad6b5f923735e24a1494f60 Mon Sep 17 00:00:00 2001 From: Maksym Aryefyev Date: Sun, 15 Sep 2024 19:40:11 +0300 Subject: [PATCH 1/2] add HugePage info to memory and topology Signed-off-by: Maksym Aryefyev --- pkg/memory/memory.go | 19 ++- pkg/memory/memory_linux.go | 174 ++++++++++++++++++++++++---- pkg/memory/memory_test.go | 8 ++ pkg/topology/topology_linux_test.go | 6 + 4 files changed, 180 insertions(+), 27 deletions(-) diff --git a/pkg/memory/memory.go b/pkg/memory/memory.go index 81e0dc66..8b316581 100644 --- a/pkg/memory/memory.go +++ b/pkg/memory/memory.go @@ -29,6 +29,15 @@ type Module struct { Vendor string `json:"vendor"` } +// HugePage describes huge page info +type HugePage struct { + Total int64 `json:"total"` + Free int64 `json:"free"` + Surplus int64 `json:"surplus"` + // Note: this field will not be populated for Topology call, since data not present in NUMA folder structure + Reserved int64 `json:"reserved"` +} + // Area describes a set of physical memory on a host system. Non-NUMA systems // will almost always have a single memory area containing all memory the // system can use. NUMA systems will have multiple memory areas, one or more @@ -37,8 +46,14 @@ type Area struct { TotalPhysicalBytes int64 `json:"total_physical_bytes"` TotalUsableBytes int64 `json:"total_usable_bytes"` // An array of sizes, in bytes, of memory pages supported in this area - SupportedPageSizes []uint64 `json:"supported_page_sizes"` - Modules []*Module `json:"modules"` + SupportedPageSizes []uint64 `json:"supported_page_sizes"` + // Default system huge pages size, in bytes + DefaultHugePageSize uint64 `json:"default_huge_page_size"` + // Amount of memory, in bytes, consumed by huge pages of all sizes. + HugeTLBSize int64 `json:"huge_tlb_size"` + // Huge page info by size + HugePages map[uint64]*HugePage `json:"huge_pages"` + Modules []*Module `json:"modules"` } // String returns a short string with a summary of information for this memory diff --git a/pkg/memory/memory_linux.go b/pkg/memory/memory_linux.go index d5a54101..638daefb 100644 --- a/pkg/memory/memory_linux.go +++ b/pkg/memory/memory_linux.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "regexp" "strconv" @@ -57,6 +58,17 @@ func (i *Info) load() error { i.TotalPhysicalBytes = tub } i.SupportedPageSizes, _ = memorySupportedPageSizes(paths.SysKernelMMHugepages) + i.DefaultHugePageSize, _ = memoryDefaultHPSizeFromPath(paths.ProcMeminfo) + i.HugeTLBSize, _ = memoryHugeTLBFromPath(paths.ProcMeminfo) + hugePage := make(map[uint64]*HugePage) + for _, p := range i.SupportedPageSizes { + info, err := memoryHPInfo(paths.SysKernelMMHugepages, p) + if err != nil { + return err + } + hugePage[p] = info + } + i.HugePages = hugePage return nil } @@ -99,10 +111,32 @@ func AreaForNode(ctx *context.Context, nodeID int) (*Area, error) { return nil, err } + defHPSize, err := memoryDefaultHPSizeFromPath(paths.ProcMeminfo) + if err != nil { + return nil, err + } + + hugeTlb, err := memoryHugeTLBFromPath(paths.ProcMeminfo) + if err != nil { + return nil, err + } + + hugePages := make(map[uint64]*HugePage) + for _, p := range supportedHP { + info, err := memoryHPInfo(filepath.Join(path, "hugepages"), p) + if err != nil { + return nil, err + } + hugePages[p] = info + } + return &Area{ - TotalPhysicalBytes: totPhys, - TotalUsableBytes: totUsable, - SupportedPageSizes: supportedHP, + TotalPhysicalBytes: totPhys, + TotalUsableBytes: totUsable, + SupportedPageSizes: supportedHP, + DefaultHugePageSize: defHPSize, + HugeTLBSize: hugeTlb, + HugePages: hugePages, }, nil } @@ -243,7 +277,113 @@ func memTotalUsableBytes(paths *linuxpath.Paths) int64 { return amount } +func memorySupportedPageSizes(hpDir string) ([]uint64, error) { + // In Linux, /sys/kernel/mm/hugepages contains a directory per page size + // supported by the kernel. The directory name corresponds to the pattern + // 'hugepages-{pagesize}kb' + out := make([]uint64, 0) + + files, err := os.ReadDir(hpDir) + if err != nil { + return out, err + } + for _, file := range files { + parts := strings.Split(file.Name(), "-") + sizeStr := parts[1] + // Cut off the 'kb' + sizeStr = sizeStr[0 : len(sizeStr)-2] + size, err := strconv.Atoi(sizeStr) + if err != nil { + return out, err + } + out = append(out, uint64(size*int(unitutil.KB))) + } + return out, nil +} + +func memoryHPInfo(hpDir string, sizeBytes uint64) (*HugePage, error) { + // In linux huge page info can be obtained in several places + // /sys/kernel/mm/hugepages/hugepages-{pagesize}kb/ directory, which contains + // nr_hugepages + // nr_hugepages_mempolicy + // nr_overcommit_hugepages + // free_hugepages + // resv_hugepages + // surplus_hugepages + // or NUMA specific data /sys/devices/system/node/node[0-9]*/hugepages/hugepages-{pagesize}kb/, which contains + // nr_hugepages + // free_hugepages + // surplus_hugepages + targetPath := filepath.Join(hpDir, fmt.Sprintf("hugepages-%vkB", sizeBytes/uint64(unitutil.KB))) + files, err := os.ReadDir(targetPath) + if err != nil { + return nil, err + } + + var ( + total int64 + free int64 + surplus int64 + reserved int64 + ) + + for _, f := range files { + switch f.Name() { + case "nr_hugepages": + count, err := readFileToInt64(path.Join(targetPath, f.Name())) + if err != nil { + return nil, err + } + total = count + case "free_hugepages": + count, err := readFileToInt64(path.Join(targetPath, f.Name())) + if err != nil { + return nil, err + } + free = count + case "surplus_hugepages": + count, err := readFileToInt64(path.Join(targetPath, f.Name())) + if err != nil { + return nil, err + } + surplus = count + case "resv_hugepages": + count, err := readFileToInt64(path.Join(targetPath, f.Name())) + if err != nil { + return nil, err + } + reserved = count + } + } + + return &HugePage{ + Total: total, + Free: free, + Surplus: surplus, + Reserved: reserved, + }, nil +} + func memoryTotalUsableBytesFromPath(meminfoPath string) (int64, error) { + const key = "MemTotal" + return getMemInfoField(meminfoPath, key) +} + +func memoryDefaultHPSizeFromPath(meminfoPath string) (uint64, error) { + const key = "Hugepagesize" + got, err := getMemInfoField(meminfoPath, key) + if err != nil { + return 0, err + } + return uint64(got), nil +} + +func memoryHugeTLBFromPath(meminfoPath string) (int64, error) { + const key = "Hugetlb" + return getMemInfoField(meminfoPath, key) +} + +func getMemInfoField(meminfoPath string, wantKey string) (int64, error) { // In Linux, /proc/meminfo or its close relative // /sys/devices/system/node/node*/meminfo // contains a set of memory-related amounts, with @@ -280,7 +420,7 @@ func memoryTotalUsableBytesFromPath(meminfoPath string) (int64, error) { line := scanner.Text() parts := strings.Split(line, ":") key := parts[0] - if !strings.Contains(key, "MemTotal") { + if !strings.Contains(key, wantKey) { continue } rawValue := parts[1] @@ -294,29 +434,13 @@ func memoryTotalUsableBytesFromPath(meminfoPath string) (int64, error) { } return int64(value), nil } - return -1, fmt.Errorf("failed to find MemTotal entry in path %q", meminfoPath) + return -1, fmt.Errorf("failed to find '%s' entry in path %q", wantKey, meminfoPath) } -func memorySupportedPageSizes(hpDir string) ([]uint64, error) { - // In Linux, /sys/kernel/mm/hugepages contains a directory per page size - // supported by the kernel. The directory name corresponds to the pattern - // 'hugepages-{pagesize}kb' - out := make([]uint64, 0) - - files, err := os.ReadDir(hpDir) +func readFileToInt64(filename string) (int64, error) { + data, err := os.ReadFile(filename) if err != nil { - return out, err - } - for _, file := range files { - parts := strings.Split(file.Name(), "-") - sizeStr := parts[1] - // Cut off the 'kb' - sizeStr = sizeStr[0 : len(sizeStr)-2] - size, err := strconv.Atoi(sizeStr) - if err != nil { - return out, err - } - out = append(out, uint64(size*int(unitutil.KB))) + return -1, err } - return out, nil + return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) } diff --git a/pkg/memory/memory_test.go b/pkg/memory/memory_test.go index 74da23da..dc3f93d0 100644 --- a/pkg/memory/memory_test.go +++ b/pkg/memory/memory_test.go @@ -45,4 +45,12 @@ func TestMemory(t *testing.T) { if len(sps) == 0 { t.Fatalf("Expected >0 supported page sizes, but got 0.") } + + if mem.DefaultHugePageSize == 0 { + t.Fatalf("Expected >0 default hugepagesize, but got 0") + } + + if len(sps) != len(mem.HugePages) { + t.Fatalf("Expected %d hugepages, but got %d", len(sps), len(mem.HugePages)) + } } diff --git a/pkg/topology/topology_linux_test.go b/pkg/topology/topology_linux_test.go index f48f399a..4eae1874 100644 --- a/pkg/topology/topology_linux_test.go +++ b/pkg/topology/topology_linux_test.go @@ -141,5 +141,11 @@ func TestTopologyPerNUMAMemory(t *testing.T) { if node.Memory.TotalUsableBytes > node.Memory.TotalPhysicalBytes { t.Fatalf("excessive usable size for node %d", node.ID) } + if node.Memory.DefaultHugePageSize == 0 { + t.Fatalf("unexpected default HP size for node %d", node.ID) + } + if len(node.Memory.HugePages) != 2 { + t.Fatalf("expected 2 huge page info records, but got '%d' for node %d", len(node.Memory.HugePages), node.ID) + } } } From 50a6480a33df268ed0ed749c566bbca49a8da917 Mon Sep 17 00:00:00 2001 From: Maksym Aryefyev Date: Wed, 18 Sep 2024 09:34:05 +0300 Subject: [PATCH 2/2] review fixes: renaming fields Signed-off-by: Maksym Aryefyev --- pkg/memory/memory.go | 14 +++++++------- pkg/memory/memory_linux.go | 30 ++++++++++++++--------------- pkg/memory/memory_test.go | 4 ++-- pkg/topology/topology_linux_test.go | 4 ++-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/memory/memory.go b/pkg/memory/memory.go index 8b316581..f58ba8b9 100644 --- a/pkg/memory/memory.go +++ b/pkg/memory/memory.go @@ -29,8 +29,8 @@ type Module struct { Vendor string `json:"vendor"` } -// HugePage describes huge page info -type HugePage struct { +// HugePageAmounts describes huge page info +type HugePageAmounts struct { Total int64 `json:"total"` Free int64 `json:"free"` Surplus int64 `json:"surplus"` @@ -47,13 +47,13 @@ type Area struct { TotalUsableBytes int64 `json:"total_usable_bytes"` // An array of sizes, in bytes, of memory pages supported in this area SupportedPageSizes []uint64 `json:"supported_page_sizes"` - // Default system huge pages size, in bytes + // Default system huge page size, in bytes DefaultHugePageSize uint64 `json:"default_huge_page_size"` - // Amount of memory, in bytes, consumed by huge pages of all sizes. - HugeTLBSize int64 `json:"huge_tlb_size"` + // Amount of memory, in bytes, consumed by huge pages of all sizes + TotalHugePageBytes int64 `json:"total_huge_page_bytes"` // Huge page info by size - HugePages map[uint64]*HugePage `json:"huge_pages"` - Modules []*Module `json:"modules"` + HugePageAmountsBySize map[uint64]*HugePageAmounts `json:"huge_page_amounts_by_size"` + Modules []*Module `json:"modules"` } // String returns a short string with a summary of information for this memory diff --git a/pkg/memory/memory_linux.go b/pkg/memory/memory_linux.go index 638daefb..53acfd5c 100644 --- a/pkg/memory/memory_linux.go +++ b/pkg/memory/memory_linux.go @@ -59,16 +59,16 @@ func (i *Info) load() error { } i.SupportedPageSizes, _ = memorySupportedPageSizes(paths.SysKernelMMHugepages) i.DefaultHugePageSize, _ = memoryDefaultHPSizeFromPath(paths.ProcMeminfo) - i.HugeTLBSize, _ = memoryHugeTLBFromPath(paths.ProcMeminfo) - hugePage := make(map[uint64]*HugePage) + i.TotalHugePageBytes, _ = memoryHugeTLBFromPath(paths.ProcMeminfo) + hugePageAmounts := make(map[uint64]*HugePageAmounts) for _, p := range i.SupportedPageSizes { info, err := memoryHPInfo(paths.SysKernelMMHugepages, p) if err != nil { return err } - hugePage[p] = info + hugePageAmounts[p] = info } - i.HugePages = hugePage + i.HugePageAmountsBySize = hugePageAmounts return nil } @@ -116,27 +116,27 @@ func AreaForNode(ctx *context.Context, nodeID int) (*Area, error) { return nil, err } - hugeTlb, err := memoryHugeTLBFromPath(paths.ProcMeminfo) + totHPSize, err := memoryHugeTLBFromPath(paths.ProcMeminfo) if err != nil { return nil, err } - hugePages := make(map[uint64]*HugePage) + hugePageAmounts := make(map[uint64]*HugePageAmounts) for _, p := range supportedHP { info, err := memoryHPInfo(filepath.Join(path, "hugepages"), p) if err != nil { return nil, err } - hugePages[p] = info + hugePageAmounts[p] = info } return &Area{ - TotalPhysicalBytes: totPhys, - TotalUsableBytes: totUsable, - SupportedPageSizes: supportedHP, - DefaultHugePageSize: defHPSize, - HugeTLBSize: hugeTlb, - HugePages: hugePages, + TotalPhysicalBytes: totPhys, + TotalUsableBytes: totUsable, + SupportedPageSizes: supportedHP, + DefaultHugePageSize: defHPSize, + TotalHugePageBytes: totHPSize, + HugePageAmountsBySize: hugePageAmounts, }, nil } @@ -301,7 +301,7 @@ func memorySupportedPageSizes(hpDir string) ([]uint64, error) { return out, nil } -func memoryHPInfo(hpDir string, sizeBytes uint64) (*HugePage, error) { +func memoryHPInfo(hpDir string, sizeBytes uint64) (*HugePageAmounts, error) { // In linux huge page info can be obtained in several places // /sys/kernel/mm/hugepages/hugepages-{pagesize}kb/ directory, which contains // nr_hugepages @@ -356,7 +356,7 @@ func memoryHPInfo(hpDir string, sizeBytes uint64) (*HugePage, error) { } } - return &HugePage{ + return &HugePageAmounts{ Total: total, Free: free, Surplus: surplus, diff --git a/pkg/memory/memory_test.go b/pkg/memory/memory_test.go index dc3f93d0..615e5682 100644 --- a/pkg/memory/memory_test.go +++ b/pkg/memory/memory_test.go @@ -50,7 +50,7 @@ func TestMemory(t *testing.T) { t.Fatalf("Expected >0 default hugepagesize, but got 0") } - if len(sps) != len(mem.HugePages) { - t.Fatalf("Expected %d hugepages, but got %d", len(sps), len(mem.HugePages)) + if len(sps) != len(mem.HugePageAmountsBySize) { + t.Fatalf("Expected %d hugepages, but got %d", len(sps), len(mem.HugePageAmountsBySize)) } } diff --git a/pkg/topology/topology_linux_test.go b/pkg/topology/topology_linux_test.go index 4eae1874..4c705e75 100644 --- a/pkg/topology/topology_linux_test.go +++ b/pkg/topology/topology_linux_test.go @@ -144,8 +144,8 @@ func TestTopologyPerNUMAMemory(t *testing.T) { if node.Memory.DefaultHugePageSize == 0 { t.Fatalf("unexpected default HP size for node %d", node.ID) } - if len(node.Memory.HugePages) != 2 { - t.Fatalf("expected 2 huge page info records, but got '%d' for node %d", len(node.Memory.HugePages), node.ID) + if len(node.Memory.HugePageAmountsBySize) != 2 { + t.Fatalf("expected 2 huge page info records, but got '%d' for node %d", len(node.Memory.HugePageAmountsBySize), node.ID) } } }