add readGlobalPerCpu interface (#291)

Summary:

make BPerfEventsGroup be able to report per CPU global value

we will need this for dynolog host-level perf counters reporting

Reviewed By: liu-song-6

Differential Revision: D61691843

hbt/src/perf_event/BPerfEventsGroup.cpp

@@ -246,8 +246,8 @@ bool BPerfEventsGroup::isOpen() const {
   HBT_DCHECK(leader_link_fd_ >= 0);
   // set proper offset_
   ::memset(offsets_, 0, sizeof(offsets_));
-  readGlobal(offsets_, true);
   enabled_ = true;
+  readGlobal(offsets_, true);
   return true;
 }
 
@@ -504,23 +504,35 @@ int BPerfEventsGroup::preparePerThreadBPerf_(bperf_leader_cgroup* skel) {
   return err;
 }
 
-int BPerfEventsGroup::read(
-    struct bpf_perf_event_value* output,
+std::vector<struct bpf_perf_event_value> BPerfEventsGroup::readFromBpf_(
     int fd,
-    __u64 id,
-    bool skip_offset) {
-  auto event_cnt = confs_.size();
+    __u64 id) {
   std::vector<struct bpf_perf_event_value> values(
       (size_t)cpu_cnt_ * BPERF_MAX_GROUP_SIZE);
-
   if (!enabled_) {
-    return -1;
+    return {};
   }
 
   syncGlobal_();
   if (int ret = ::bpf_map_lookup_elem(fd, &id, values.data()); ret) {
     HBT_LOG_ERROR() << "cannot look up key " << id
                     << " from output map. Return value: " << ret;
+    return {};
+  }
+
+  return values;
+}
+
+int BPerfEventsGroup::read(
+    struct bpf_perf_event_value* output,
+    int fd,
+    __u64 id,
+    bool skip_offset) {
+  auto event_cnt = confs_.size();
+
+  auto values = readFromBpf_(fd, id);
+
+  if (values.empty()) {
     return -1;
   }
 
@@ -542,6 +554,28 @@ int BPerfEventsGroup::read(
   return event_cnt;
 }
 
+int BPerfEventsGroup::readPerCpu(
+    std::map<
+        int,
+        std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>& output,
+    int fd,
+    __u64 id) {
+  auto event_cnt = confs_.size();
+
+  auto values = readFromBpf_(fd, id);
+
+  for (size_t e = 0; e < event_cnt; e++) {
+    for (int c = 0; c < cpu_cnt_; c++) {
+      int idx = c * BPERF_MAX_GROUP_SIZE + e;
+      output[c][e].counter = values[idx].counter;
+      output[c][e].enabled = values[idx].enabled;
+      output[c][e].running = values[idx].running;
+    }
+  }
+
+  return event_cnt;
+}
+
 int BPerfEventsGroup::readGlobal(
     struct bpf_perf_event_value* output,
     bool skip_offset) {
@@ -561,6 +595,24 @@ bool BPerfEventsGroup::readGlobal(ReadValues& rv, bool skip_offset) {
   }
 }
 
+int BPerfEventsGroup::readGlobalPerCpu(
+    std::
+        map<int, std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>&
+            output) {
+  return readPerCpu(output, global_output_fd_, 0);
+}
+
+bool BPerfEventsGroup::readGlobalPerCpu(std::map<int, ReadValues>& rv) {
+  std::map<int, std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>
+      output;
+  int numEvents = readGlobalPerCpu(output);
+  for (auto& [cpu, values] : output) {
+    rv.insert({cpu, ReadValues(numEvents)});
+    toReadValues(rv.at(cpu), values.data());
+  }
+  return numEvents > 0;
+}
+
 int BPerfEventsGroup::readCgroup(
     struct bpf_perf_event_value* output,
     __u64 id) {

hbt/src/perf_event/BPerfEventsGroup.h

@@ -72,6 +72,12 @@ class BPerfEventsGroup {
   // eBPF like interface to read counters from all CPUs and accumulate them.
   int readGlobal(struct bpf_perf_event_value* output, bool skip_offset = false);
   bool readGlobal(ReadValues& rv, bool skip_offset = false);
+  int readGlobalPerCpu(
+      std::map<
+          int,
+          std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>&
+          output);
+  bool readGlobalPerCpu(std::map<int, ReadValues>& rv);
 
   int readCgroup(struct bpf_perf_event_value* output, __u64 id);
   bool readCgroup(ReadValues& rv, __u64 id);
@@ -122,12 +128,22 @@ class BPerfEventsGroup {
   //     read(&offset_);
   struct bpf_perf_event_value offsets_[BPERF_MAX_GROUP_SIZE];
 
+  std::vector<struct bpf_perf_event_value> readFromBpf_(int fd, __u64 id);
+
   int read(
       struct bpf_perf_event_value* output,
       int fd,
       __u64 id,
       bool skip_offset = false);
 
+  int readPerCpu(
+      std::map<
+          int,
+          std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>&
+          output,
+      int fd,
+      __u64 id);
+
   int reloadSkel_(struct bperf_attr_map_elem* entry);
   int loadPerfEvent_(struct bperf_leader_cgroup* skel);
 

hbt/src/perf_event/tests/BPerfEventsGroupTest.cpp

@@ -55,6 +55,15 @@ TEST(BPerfEventsGroupTest, RunSystemWide) {
     checkReading(val, prev, n);
     ::memcpy(prev, val, sizeof(prev));
   }
+
+  std::map<int, std::array<struct bpf_perf_event_value, BPERF_MAX_GROUP_SIZE>>
+      perCpuValues;
+  auto n = system.readGlobalPerCpu(perCpuValues);
+  EXPECT_GT(n, 0);
+  EXPECT_EQ(perCpuValues.size(), CpuSet::makeAllOnline().numCpus());
+  for (const auto& [cpu, values] : perCpuValues) {
+    EXPECT_GE(values[0].counter, 0);
+  }
 }
 
 TEST(BPerfEventsGroupTest, RunCgroup) {