santa-driver: add back the root and non-root caches (#302)

* santa-driver: add back the root and non-root caches

* cachehistogram: clarify buckets and entries

* review changes

Source/common/SNTXPCUnprivilegedControlInterface.h

@@ -33,7 +33,7 @@
 ///
 ///  Kernel ops
 ///
-- (void)cacheCounts:(void (^)(uint64_t count))reply;
+- (void)cacheCounts:(void (^)(uint64_t rootCache, uint64_t nonRootCache))reply;
 - (void)cacheBucketCount:(void (^)(NSArray *))reply;
 - (void)checkCacheForVnodeID:(santa_vnode_id_t)vnodeID withReply:(void (^)(santa_action_t))reply;
 - (void)driverConnectionEstablished:(void (^)(BOOL))reply;

Source/santa-driver/SantaDecisionManager.cc

@@ -41,7 +41,8 @@ bool SantaDecisionManager::init() {
   decision_dataqueue_lock_ = lck_mtx_alloc_init(sdm_lock_grp_, sdm_lock_attr_);
   log_dataqueue_lock_ = lck_mtx_alloc_init(sdm_lock_grp_, sdm_lock_attr_);
 
-  decision_cache_ = new SantaCache<santa_vnode_id_t, uint64_t>(10000, 2);
+  root_decision_cache_ = new SantaCache<santa_vnode_id_t, uint64_t>(5000, 2);
+  non_root_decision_cache_ = new SantaCache<santa_vnode_id_t, uint64_t>(500, 2);
   vnode_pid_map_ = new SantaCache<santa_vnode_id_t, uint64_t>(2000, 5);
   compiler_pid_set_ = new SantaCache<pid_t, pid_t>(500, 5);
 
@@ -54,12 +55,14 @@ bool SantaDecisionManager::init() {
   if (!log_dataqueue_) return kIOReturnNoMemory;
 
   client_pid_ = 0;
+  root_fsid_ = 0;
 
   return true;
 }
 
 void SantaDecisionManager::free() {
-  delete decision_cache_;
+  delete root_decision_cache_;
+  delete non_root_decision_cache_;
   delete vnode_pid_map_;
 
   StopPidMonitorThreads();
@@ -102,6 +105,17 @@ void SantaDecisionManager::ConnectClient(pid_t pid) {
 
   client_pid_ = pid;
 
+  // Determine root fsid
+  vfs_context_t ctx = vfs_context_create(nullptr);
+  if (ctx) {
+    vnode_t root = vfs_rootvnode();
+    if (root) {
+      root_fsid_ = GetVnodeIDForVnode(ctx, root).fsid;
+      vnode_put(root);
+    }
+    vfs_context_rele(ctx);
+  }
+
   // Any decisions made while the daemon wasn't
   // connected should be cleared
   ClearCache();
@@ -292,30 +306,43 @@ uint32_t SantaDecisionManager::PidMonitorSleepTimeMilliseconds() const {
 
 #pragma mark Cache Management
 
+/**
+ Return the correct cache for a given identifier.
+
+ @param identifier The identifier
+ @return SantaCache* The cache to use
+*/
+SantaCache<santa_vnode_id_t, uint64_t> *SantaDecisionManager::CacheForIdentifier(
+    const santa_vnode_id_t identifier) {
+  return (identifier.fsid == root_fsid_) ? root_decision_cache_ : non_root_decision_cache_;
+}
+
 void SantaDecisionManager::AddToCache(
     santa_vnode_id_t identifier, santa_action_t decision, uint64_t microsecs) {
+  auto decision_cache = CacheForIdentifier(identifier);
+
   switch (decision) {
     case ACTION_REQUEST_BINARY:
-      decision_cache_->set(identifier, (uint64_t)ACTION_REQUEST_BINARY << 56, 0);
+      decision_cache->set(identifier, (uint64_t)ACTION_REQUEST_BINARY << 56, 0);
       break;
     case ACTION_RESPOND_ACK:
-      decision_cache_->set(identifier, (uint64_t)ACTION_RESPOND_ACK << 56,
+      decision_cache->set(identifier, (uint64_t)ACTION_RESPOND_ACK << 56,
                           ((uint64_t)ACTION_REQUEST_BINARY << 56));
       break;
     case ACTION_RESPOND_ALLOW:
     case ACTION_RESPOND_ALLOW_COMPILER:
     case ACTION_RESPOND_DENY: {
       // Decision is stored in upper 8 bits, timestamp in remaining 56.
       uint64_t val = ((uint64_t)decision << 56) | (microsecs & 0xFFFFFFFFFFFFFF);
-      if (!decision_cache_->set(identifier, val, ((uint64_t)ACTION_REQUEST_BINARY << 56))) {
-        decision_cache_->set(identifier, val, ((uint64_t)ACTION_RESPOND_ACK << 56));
+      if (!decision_cache->set(identifier, val, ((uint64_t)ACTION_REQUEST_BINARY << 56))) {
+        decision_cache->set(identifier, val, ((uint64_t)ACTION_RESPOND_ACK << 56));
       }
       break;
     }
     case ACTION_RESPOND_ALLOW_PENDING_TRANSITIVE: {
       // Decision is stored in upper 8 bits, timestamp in remaining 56.
       uint64_t val = ((uint64_t)decision << 56) | (microsecs & 0xFFFFFFFFFFFFFF);
-      decision_cache_->set(identifier, val, 0);
+      decision_cache->set(identifier, val, 0);
       break;
     }
     default:
@@ -327,21 +354,26 @@ void SantaDecisionManager::AddToCache(
 
 void SantaDecisionManager::RemoveFromCache(santa_vnode_id_t identifier) {
   if (unlikely(identifier.fsid == 0 && identifier.fileid == 0)) return;
-  decision_cache_->remove(identifier);
+  CacheForIdentifier(identifier)->remove(identifier);
   wakeup((void *)identifier.unsafe_simple_id());
 }
 
-uint64_t SantaDecisionManager::CacheCount() const {
-  return decision_cache_->count();
+uint64_t SantaDecisionManager::RootCacheCount() const {
+  return root_decision_cache_->count();
 }
 
-void SantaDecisionManager::ClearCache() {
-  decision_cache_->clear();
+uint64_t SantaDecisionManager::NonRootCacheCount() const {
+  return non_root_decision_cache_->count();
+}
+
+void SantaDecisionManager::ClearCache(bool non_root_only) {
+  if (!non_root_only) root_decision_cache_->clear();
+  non_root_decision_cache_->clear();
 }
 
 void SantaDecisionManager::CacheBucketCount(
     uint16_t *per_bucket_counts, uint16_t *array_size, uint64_t *start_bucket) {
-  decision_cache_->bucket_counts(per_bucket_counts, array_size, start_bucket);
+  root_decision_cache_->bucket_counts(per_bucket_counts, array_size, start_bucket);
 }
 
 #pragma mark Decision Fetching
@@ -350,7 +382,9 @@ santa_action_t SantaDecisionManager::GetFromCache(santa_vnode_id_t identifier) {
   auto result = ACTION_UNSET;
   uint64_t decision_time = 0;
 
-  uint64_t cache_val = decision_cache_->get(identifier);
+  auto decision_cache = CacheForIdentifier(identifier);
+
+  uint64_t cache_val = decision_cache->get(identifier);
   if (cache_val == 0) return result;
 
   // Decision is stored in upper 8 bits, timestamp in remaining 56.
@@ -361,7 +395,7 @@ santa_action_t SantaDecisionManager::GetFromCache(santa_vnode_id_t identifier) {
     if (result == ACTION_RESPOND_DENY) {
       auto expiry_time = decision_time + (kMaxDenyCacheTimeMilliseconds * 1000);
       if (expiry_time < GetCurrentUptime()) {
-        decision_cache_->remove(identifier);
+        decision_cache->remove(identifier);
         return ACTION_UNSET;
       }
     }

Source/santa-driver/SantaDecisionManager.h

@@ -123,11 +123,14 @@ class SantaDecisionManager : public OSObject {
   void RemoveFromCache(santa_vnode_id_t identifier);
 
   /// Returns the number of entries in the cache.
-  uint64_t CacheCount() const;
-
-  /// Clears the cache.
-  void ClearCache();
+  uint64_t RootCacheCount() const;
+  uint64_t NonRootCacheCount() const;
 
+  /**
+   Clears the cache(s). If non_root_only is true, only the non-root cache
+   is cleared.
+  */
+  void ClearCache(bool non_root_only = false);
 
   /**
     Fills out the per_bucket_counts array with the number of items in each bucket in the
@@ -325,10 +328,23 @@ class SantaDecisionManager : public OSObject {
     return (uint64_t)((sec * 1000000) + usec);
   }
 
-  SantaCache<santa_vnode_id_t, uint64_t> *decision_cache_;
+  SantaCache<santa_vnode_id_t, uint64_t> *root_decision_cache_;
+  SantaCache<santa_vnode_id_t, uint64_t> *non_root_decision_cache_;
   SantaCache<santa_vnode_id_t, uint64_t> *vnode_pid_map_;
   SantaCache<pid_t, pid_t> *compiler_pid_set_;
 
+  /**
+   Return the correct cache for a given identifier.
+
+   @param identifier The identifier
+   @return SantaCache* The cache to use
+  */
+  SantaCache<santa_vnode_id_t, uint64_t>* CacheForIdentifier(const santa_vnode_id_t identifier);
+
+  // This is the file system ID of the root filesystem,
+  // used to determine which cache to use for requests
+  uint64_t root_fsid_;
+
   lck_grp_t *sdm_lock_grp_;
   lck_grp_attr_t *sdm_lock_grp_attr_;
   lck_attr_t *sdm_lock_attr_;

Source/santa-driver/SantaDriverClient.cc

@@ -189,7 +189,8 @@ IOReturn SantaDriverClient::clear_cache(
   SantaDriverClient *me = OSDynamicCast(SantaDriverClient, target);
   if (!me) return kIOReturnBadArgument;
 
-  me->decisionManager->ClearCache();
+  const bool non_root_only = static_cast<const bool>(arguments->scalarInput[0]);
+  me->decisionManager->ClearCache(non_root_only);
 
   return kIOReturnSuccess;
 }
@@ -212,7 +213,8 @@ IOReturn SantaDriverClient::cache_count(
   SantaDriverClient *me = OSDynamicCast(SantaDriverClient, target);
   if (!me) return kIOReturnBadArgument;
 
-  arguments->scalarOutput[0] = me->decisionManager->CacheCount();
+  arguments->scalarOutput[0] = me->decisionManager->RootCacheCount();
+  arguments->scalarOutput[1] = me->decisionManager->NonRootCacheCount();
   return kIOReturnSuccess;
 }
 
@@ -263,9 +265,9 @@ IOReturn SantaDriverClient::externalMethod(
     { &SantaDriverClient::allow_compiler, 0, sizeof(santa_vnode_id_t), 0, 0 },
     { &SantaDriverClient::deny_binary, 0, sizeof(santa_vnode_id_t), 0, 0 },
     { &SantaDriverClient::acknowledge_binary, 0, sizeof(santa_vnode_id_t), 0, 0 },
-    { &SantaDriverClient::clear_cache, 0, 0, 0, 0 },
+    { &SantaDriverClient::clear_cache, 1, 0, 0, 0 },
     { &SantaDriverClient::remove_cache_entry, 0, sizeof(santa_vnode_id_t), 0, 0 },
-    { &SantaDriverClient::cache_count, 0, 0, 1, 0 },
+    { &SantaDriverClient::cache_count, 0, 0, 2, 0 },
     { &SantaDriverClient::check_cache, 0, sizeof(santa_vnode_id_t), 1, 0 },
     { &SantaDriverClient::cache_bucket_count, 0, sizeof(santa_bucket_count_t),
         0, sizeof(santa_bucket_count_t) },

Source/santactl/Commands/SNTCommandCacheHistogram.m

@@ -69,7 +69,7 @@ - (void)runWithArguments:(NSArray *)arguments {
         }
         printf("\n");
       } else {
-        printf("%4llu: %llu\n", k, v);
+        printf("%4llu bucket[s] have %llu %s\n", v, k, k > 1 ? "entries" : "entry");
       }
     }
     exit(0);

Source/santactl/Commands/SNTCommandStatus.m

@@ -87,10 +87,11 @@ - (void)runWithArguments:(NSArray *)arguments {
   BOOL fileLogging = ([[SNTConfigurator configurator] fileChangesRegex] != nil);
 
   // Kext status
-  __block uint64_t cacheCount = -1;
+  __block uint64_t rootCacheCount = -1, nonRootCacheCount = -1;
   dispatch_group_enter(group);
-  [[self.daemonConn remoteObjectProxy] cacheCounts:^(uint64_t count) {
-    cacheCount = count;
+  [[self.daemonConn remoteObjectProxy] cacheCounts:^(uint64_t rootCache, uint64_t nonRootCache) {
+    rootCacheCount = rootCache;
+    nonRootCacheCount = nonRootCache;
     dispatch_group_leave(group);
   }];
 
@@ -187,7 +188,8 @@ - (void)runWithArguments:(NSArray *)arguments {
         @"watchdog_ram_peak" : @(ramPeak),
       },
       @"kernel" : @{
-        @"cache_count" : @(cacheCount),
+        @"root_cache_count" : @(rootCacheCount),
+        @"non_root_cache_count": @(nonRootCacheCount),
       },
       @"database" : @{
         @"binary_rules" : @(binaryRuleCount),
@@ -219,7 +221,8 @@ - (void)runWithArguments:(NSArray *)arguments {
     printf("  %-25s | %lld  (Peak: %.2f%%)\n", "Watchdog CPU Events", cpuEvents, cpuPeak);
     printf("  %-25s | %lld  (Peak: %.2fMB)\n", "Watchdog RAM Events", ramEvents, ramPeak);
     printf(">>> Kernel Info\n");
-    printf("  %-25s | %lld\n", "Cache count", cacheCount);
+    printf("  %-25s | %lld\n", "Root cache count", rootCacheCount);
+    printf("  %-25s | %lld\n", "Non-root cache count", nonRootCacheCount);
     printf(">>> Database Info\n");
     printf("  %-25s | %lld\n", "Binary Rules", binaryRuleCount);
     printf("  %-25s | %lld\n", "Certificate Rules", certRuleCount);

Source/santad/SNTApplication.m

@@ -264,7 +264,7 @@ void diskDisappearedCallback(DADiskRef disk, void *context) {
   if (![props[@"DAVolumeMountable"] boolValue]) return;
 
   [app.eventLog logDiskDisappeared:props];
-  [app.driverManager flushCache];
+  [app.driverManager flushCacheNonRootOnly:YES];
 }
 
 - (void)startSyncd {
@@ -316,15 +316,15 @@ - (void)observeValueForKeyPath:(NSString *)keyPath
     if (!new && !old) return;
     if (![new.pattern isEqualToString:old.pattern]) {
       LOGI(@"Changed [white|black]list regex, flushing cache");
-      [self.driverManager flushCache];
+      [self.driverManager flushCacheNonRootOnly:NO];
     }
   }
 }
 
 - (void)clientModeDidChange:(SNTClientMode)clientMode {
   if (clientMode == SNTClientModeLockdown) {
     LOGI(@"Changed client mode, flushing cache.");
-    [self.driverManager flushCache];
+    [self.driverManager flushCacheNonRootOnly:NO];
   }
   [[self.notQueue.notifierConnection remoteObjectProxy] postClientModeNotification:clientMode];
 }

Source/santad/SNTDaemonControlController.m

@@ -70,17 +70,17 @@ - (instancetype)initWithDriverManager:(SNTDriverManager *)driverManager
 
 #pragma mark Kernel ops
 
-- (void)cacheCounts:(void (^)(uint64_t))reply {
-  uint64_t count = [self.driverManager cacheCount];
-  reply(count);
+- (void)cacheCounts:(void (^)(uint64_t, uint64_t))reply {
+  NSArray<NSNumber *> *counts = [self.driverManager cacheCounts];
+  reply([counts[0] unsignedLongLongValue], [counts[1] unsignedLongLongValue]);
 }
 
 - (void)cacheBucketCount:(void (^)(NSArray *))reply {
   reply([self.driverManager cacheBucketCount]);
 }
 
 - (void)flushCache:(void (^)(BOOL))reply {
-  reply([self.driverManager flushCache]);
+  reply([self.driverManager flushCacheNonRootOnly:NO]);
 }
 
 - (void)checkCacheForVnodeID:(santa_vnode_id_t)vnodeID withReply:(void (^)(santa_action_t))reply {
@@ -121,7 +121,7 @@ - (void)databaseRuleAddRules:(NSArray *)rules
   // The actual cache flushing happens after the new rules have been added to the database.
   if (flushCache) {
     LOGI(@"Flushing decision cache");
-    [self.driverManager flushCache];
+    [self.driverManager flushCacheNonRootOnly:NO];
   }
 
   reply(error);

Source/santad/SNTDriverManager.h

@@ -50,7 +50,7 @@
 ///
 ///  Get the number of binaries in the kernel's caches.
 ///
-- (uint64_t)cacheCount;
+- (NSArray<NSNumber *> *)cacheCounts;
 
 ///
 ///  Return an array representing all buckets in the kernel's decision cache where each number
@@ -61,7 +61,7 @@
 ///
 ///  Flush the kernel's binary cache.
 ///
-- (BOOL)flushCache;
+- (BOOL)flushCacheNonRootOnly:(BOOL)nonRootOnly;
 
 ///
 ///  Check the kernel cache for a VnodeID.

Source/santad/SNTDriverManager.m

@@ -213,9 +213,9 @@ - (kern_return_t)postToKernelAction:(santa_action_t)action forVnodeID:(santa_vno
   }
 }
 
-- (uint64_t)cacheCount {
-  uint32_t input_count = 1;
-  uint64_t cache_counts[1] = {0};
+- (NSArray<NSNumber *> *)cacheCounts {
+  uint32_t input_count = 2;
+  uint64_t cache_counts[2] = {0, 0};
 
   IOConnectCallScalarMethod(_connection,
                             kSantaUserClientCacheCount,
@@ -224,14 +224,16 @@ - (uint64_t)cacheCount {
                             cache_counts,
                             &input_count);
 
-  return cache_counts[0];
+  return @[ @(cache_counts[0]), @(cache_counts[1]) ];
 }
 
-- (BOOL)flushCache {
+- (BOOL)flushCacheNonRootOnly:(BOOL)nonRootOnly {
+  const uint64_t nonRoot = nonRootOnly;
+
   return IOConnectCallScalarMethod(_connection,
                                    kSantaUserClientClearCache,
-                                   0,
-                                   0,
+                                   &nonRoot,
+                                   1,
                                    0,
                                    0) == KERN_SUCCESS;
 }

Tests/KernelTests/main.mm

@@ -151,7 +151,8 @@ - (void)postToKernelAction:(santa_action_t)action forVnodeID:(santa_vnode_id_t)v
 
 /// Call in-kernel function: |kSantaUserClientClearCache|
 - (void)flushCache {
-  IOConnectCallScalarMethod(self.connection, kSantaUserClientClearCache, 0, 0, 0, 0);
+  uint64_t nonRootOnly = 0;
+  IOConnectCallScalarMethod(self.connection, kSantaUserClientClearCache, &nonRootOnly, 1, 0, 0);
 }
 
 #pragma mark - Connection Tests