santactl/sync: fixed exception when file_name is None / NSNull (#180)

* santactl/sync: fixed exception when file_name is None / NSNull

* review updates

Source/SantaGUI/Resources/MessageWindow.xib

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="3.0" toolsVersion="10117" systemVersion="16E195" targetRuntime="MacOSX.Cocoa" propertyAccessControl="none" useAutolayout="YES">
+<document type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="3.0" toolsVersion="10117" systemVersion="16F73" targetRuntime="MacOSX.Cocoa" propertyAccessControl="none" useAutolayout="YES">
     <dependencies>
         <deployment identifier="macosx"/>
         <development version="6300" identifier="xcode"/>
@@ -10,6 +10,7 @@
             <connections>
                 <outlet property="applicationNameLabel" destination="qgf-Jf-cJr" id="1JX-X8-03v"/>
                 <outlet property="bundleHashLabel" destination="xP7-jE-NF8" id="i8B-Gs-2E3"/>
+                <outlet property="bundleHashTitle" destination="MhO-U0-MLR" id="KT0-bK-fpV"/>
                 <outlet property="foundFileCountLabel" destination="LHV-gV-vyf" id="Sr0-T2-xGx"/>
                 <outlet property="hashingIndicator" destination="VyY-Yg-JOe" id="Yq4-tZ-9ep"/>
                 <outlet property="openEventButton" destination="7ua-5a-uSd" id="9s4-ZA-Vlo"/>

Source/SantaGUI/SNTMessageWindowController.h

@@ -35,6 +35,10 @@
 ///  doesn't have a bundle hash.
 @property(weak) IBOutlet NSTextField *bundleHashLabel;
 
+///  Reference to the "Bundle Identifier" label in the XIB. Used to remove if application
+///  doesn't have a bundle hash.
+@property(weak) IBOutlet NSTextField *bundleHashTitle;
+
 ///
 /// Is displayed if calculating the bundle hash is taking a bit.
 ///

Source/SantaGUI/SNTMessageWindowController.m

@@ -70,7 +70,6 @@
       NSProgress *progress = object;
       if (progress.fractionCompleted != 0.0) {
         self.hashingIndicator.indeterminate = NO;
-        [self.foundFileCountLabel removeFromSuperview];
       }
       self.hashingIndicator.doubleValue = progress.fractionCompleted;
     });

Source/SantaGUI/SNTNotificationManager.m

@@ -178,11 +178,13 @@ static NSString * const silencedNotificationsKey = @"SilencedNotifications";
 
 - (void)updateCountsForEvent:(SNTStoredEvent *)event
                  binaryCount:(uint64_t)binaryCount
-                   fileCount:(uint64_t)fileCount {
+                   fileCount:(uint64_t)fileCount
+                 hashedCount:(uint64_t)hashedCount {
   if ([self.currentWindowController.event.idx isEqual:event.idx]) {
     dispatch_async(dispatch_get_main_queue(), ^{
       self.currentWindowController.foundFileCountLabel.stringValue =
-          [NSString stringWithFormat:@"%llu binaries / %llu files", binaryCount, fileCount];
+          [NSString stringWithFormat:@"%llu binaries / %llu %@",
+               binaryCount, hashedCount ?: fileCount, hashedCount ? @"hashed" : @"files"];
     });
   }
 }
@@ -192,12 +194,23 @@ static NSString * const silencedNotificationsKey = @"SilencedNotifications";
   c.remoteInterface = [SNTXPCBundleServiceInterface bundleServiceInterface];
   [c resume];
   self.bundleServiceConnection = c;
+
+  WEAKIFY(self);
+  self.bundleServiceConnection.invalidationHandler = ^{
+    STRONGIFY(self);
+    if (self.currentWindowController) {
+      [self updateBlockNotification:self.currentWindowController.event withBundleHash:nil];
+    }
+  };
+
   dispatch_semaphore_signal(self.bundleServiceSema);
 }
 
 #pragma mark SNTBundleNotifierXPC helper methods
 
 - (void)hashBundleBinariesForEvent:(SNTStoredEvent *)event {
+  self.currentWindowController.foundFileCountLabel.stringValue = @"Searching for files...";
+
   // Wait a max of 6 secs for the bundle service. Should the bundle service fall over, it will
   // reconnect within 5 secs. Otherwise abandon bundle hashing and display the blockable event.
   if (dispatch_semaphore_wait(self.bundleServiceSema,
@@ -210,7 +223,7 @@ static NSString * const silencedNotificationsKey = @"SilencedNotifications";
   dispatch_semaphore_signal(self.bundleServiceSema);
 
   // NSProgress becomes current for this thread. XPC messages vend a child node to the receiver.
-  [self.currentWindowController.progress becomeCurrentWithPendingUnitCount:1];
+  [self.currentWindowController.progress becomeCurrentWithPendingUnitCount:100];
 
   // Start hashing. Progress is reported to the root NSProgress (currentWindowController.progress).
   [[self.bundleServiceConnection remoteObjectProxy]
@@ -222,6 +235,7 @@ static NSString * const silencedNotificationsKey = @"SilencedNotifications";
     event.fileBundleHash = bh;
     event.fileBundleBinaryCount = @(events.count);
     event.fileBundleHashMilliseconds = ms;
+    event.fileBundleExecutableRelPath = [events.firstObject fileBundleExecutableRelPath];
     for (SNTStoredEvent *se in events) {
       se.fileBundleHash = bh;
       se.fileBundleBinaryCount = @(events.count);
@@ -246,6 +260,7 @@ static NSString * const silencedNotificationsKey = @"SilencedNotifications";
         [self.currentWindowController.bundleHashLabel setHidden:NO];
       } else {
         [self.currentWindowController.bundleHashLabel removeFromSuperview];
+        [self.currentWindowController.bundleHashTitle removeFromSuperview];
       }
       self.currentWindowController.event.fileBundleHash = bundleHash;
       [self.currentWindowController.foundFileCountLabel removeFromSuperview];

Source/common/SNTFileInfo.h

@@ -38,6 +38,18 @@
 ///
 - (instancetype)initWithPath:(NSString *)path;
 
+
+///
+///  Initializer for already resolved paths.
+///
+///  @param path The path of the file this instance is to represent. The path will
+///      not be converted and will be used as is. If the path is not a regular file this method will
+///      return nil and fill in an error.
+///  @param error If an error occurred and nil is returned, this will be a pointer to an NSError
+///      describing the problem.
+///
+- (instancetype)initWithResolvedPath:(NSString *)path error:(NSError **)error;
+
 ///
 ///  @return Path of this file.
 ///

Source/common/SNTFileInfo.m

@@ -59,23 +59,43 @@
 
 extern NSString *const NSURLQuarantinePropertiesKey WEAK_IMPORT_ATTRIBUTE;
 
-- (instancetype)initWithPath:(NSString *)path error:(NSError **)error {
+- (instancetype)initWithResolvedPath:(NSString *)path error:(NSError **)error {
   self = [super init];
   if (self) {
-    NSBundle *bndl;
-    _path = [self resolvePath:path bundle:&bndl];
-    _bundleRef = bndl;
-    if (_path.length == 0) {
+    _path = path;
+    if (!_path.length) {
       if (error) {
-        NSString *errStr = @"Unable to resolve empty path";
-        if (path) errStr = [@"Unable to resolve path: " stringByAppendingString:path];
+        NSString *errStr = @"Unable to use empty path";
         *error = [NSError errorWithDomain:@"com.google.santa.fileinfo"
-                                     code:260
+                                     code:270
                                  userInfo:@{NSLocalizedDescriptionKey : errStr}];
       }
       return nil;
     }
 
+    struct stat fileStat;
+    lstat(_path.UTF8String, &fileStat);
+    if (!((S_IFMT & fileStat.st_mode) == S_IFREG)) {
+      if (error) {
+        NSString *errStr = [NSString stringWithFormat:@"Non regular file: %s", strerror(errno)];
+        *error = [NSError errorWithDomain:@"com.google.santa.fileinfo"
+                                     code:290
+                                 userInfo:@{NSLocalizedDescriptionKey : errStr}];
+      }
+      return nil;
+    }
+
+    _fileSize = fileStat.st_size;
+
+    if (_fileSize == 0) return nil;
+
+    if (fileStat.st_uid != 0) {
+      struct passwd *pwd = getpwuid(fileStat.st_uid);
+      if (pwd) {
+        _fileOwnerHomeDir = @(pwd->pw_dir);
+      }
+    }
+
     int fd = open([_path UTF8String], O_RDONLY | O_CLOEXEC);
     if (fd < 0) {
       if (error) {
@@ -87,24 +107,29 @@ extern NSString *const NSURLQuarantinePropertiesKey WEAK_IMPORT_ATTRIBUTE;
       return nil;
     }
     _fileHandle = [[NSFileHandle alloc] initWithFileDescriptor:fd closeOnDealloc:YES];
-
-    struct stat fileStat;
-    fstat(_fileHandle.fileDescriptor, &fileStat);
-    _fileSize = fileStat.st_size;
-
-    if (_fileSize == 0) return nil;
-
-    if (fileStat.st_uid != 0) {
-      struct passwd *pwd = getpwuid(fileStat.st_uid);
-      if (pwd) {
-        _fileOwnerHomeDir = @(pwd->pw_dir);
-      }
-    }
   }
 
   return self;
 }
 
+- (instancetype)initWithPath:(NSString *)path error:(NSError **)error {
+  NSBundle *bndl;
+  NSString *resolvedPath = [self resolvePath:path bundle:&bndl];
+  if (!resolvedPath.length) {
+    if (error) {
+      NSString *errStr = @"Unable to resolve empty path";
+      if (path) errStr = [@"Unable to resolve path: " stringByAppendingString:path];
+      *error = [NSError errorWithDomain:@"com.google.santa.fileinfo"
+                                   code:260
+                               userInfo:@{NSLocalizedDescriptionKey : errStr}];
+    }
+    return nil;
+  }
+  self = [self initWithResolvedPath:resolvedPath error:error];
+  if (self && bndl) _bundleRef = bndl;
+  return self;
+}
+
 - (instancetype)initWithPath:(NSString *)path {
   return [self initWithPath:path error:NULL];
 }

Source/common/SNTStoredEvent.h

@@ -69,6 +69,11 @@
 ///
 @property NSString *fileBundlePath;
 
+///
+///  The relative path to the bundle's main executable.
+///
+@property NSString *fileBundleExecutableRelPath;
+
 ///
 ///  If the executed file was part of the bundle, this is the CFBundleID.
 ///

Source/common/SNTStoredEvent.m

@@ -39,6 +39,7 @@
   ENCODE(self.fileBundleBinaryCount, @"fileBundleBinaryCount");
   ENCODE(self.fileBundleName, @"fileBundleName");
   ENCODE(self.fileBundlePath, @"fileBundlePath");
+  ENCODE(self.fileBundleExecutableRelPath, @"fileBundleExecutableRelPath");
   ENCODE(self.fileBundleID, @"fileBundleID");
   ENCODE(self.fileBundleVersion, @"fileBundleVersion");
   ENCODE(self.fileBundleVersionString, @"fileBundleVersionString");
@@ -82,6 +83,7 @@
     _fileBundleBinaryCount = DECODE(NSNumber, @"fileBundleBinaryCount");
     _fileBundleName = DECODE(NSString, @"fileBundleName");
     _fileBundlePath = DECODE(NSString, @"fileBundlePath");
+    _fileBundleExecutableRelPath = DECODE(NSString, @"fileBundleExecutableRelPath");
     _fileBundleID = DECODE(NSString, @"fileBundleID");
     _fileBundleVersion = DECODE(NSString, @"fileBundleVersion");
     _fileBundleVersionString = DECODE(NSString, @"fileBundleVersionString");

Source/common/SNTXPCBundleServiceInterface.h

@@ -31,7 +31,8 @@ typedef void (^SNTBundleHashBlock)(NSString *, NSArray<SNTStoredEvent *> *, NSNu
 ///  Hash a bundle for an event. The SNTBundleHashBlock will be called with nil parameters if a
 ///  failure or cancellation occurs.
 ///
-///  @param event The event that includes the fileBundlePath to be hashed.
+///  @param event The event that includes the fileBundlePath to be hashed. This method will
+///      attempt to to find and use the ancestor bundle as a starting point.
 ///  @param reply A SNTBundleHashBlock to be executed upon completion or cancellation.
 ///
 ///  @note If there is a current NSProgress when called this method will report back its progress.

Source/common/SNTXPCNotifierInterface.h

@@ -30,7 +30,9 @@
 @protocol SNTBundleNotifierXPC
 - (void)updateCountsForEvent:(SNTStoredEvent *)event
                  binaryCount:(uint64_t)binaryCount
-                   fileCount:(uint64_t)fileCount;
+                   fileCount:(uint64_t)fileCount
+                 hashedCount:(uint64_t)hashedCount;
+
 - (void)setBundleServiceListener:(NSXPCListenerEndpoint *)listener;
 @end
 

Source/santabs/SNTBundleService.m

@@ -20,7 +20,6 @@
 #import "MOLCertificate.h"
 #import "MOLCodesignChecker.h"
 #import "SNTFileInfo.h"
-#import "SNTLogging.h"
 #import "SNTStoredEvent.h"
 #import "SNTXPCConnection.h"
 #import "SNTXPCNotifierInterface.h"
@@ -28,10 +27,19 @@
 @interface SNTBundleService ()
 @property SNTXPCConnection *notifierConnection;
 @property SNTXPCConnection *listener;
+@property(nonatomic) dispatch_queue_t queue;
 @end
 
 @implementation SNTBundleService
 
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0);
+  }
+  return self;
+}
+
 #pragma mark Connection handling
 
 // Create a listener for SantaGUI to connect
@@ -67,7 +75,6 @@
   [self performSelectorInBackground:@selector(createConnection) withObject:nil];
 }
 
-
 #pragma mark SNTBundleServiceXPC Methods
 
 // Connect to the SantaGUI
@@ -76,7 +83,7 @@
   c.remoteInterface = [SNTXPCNotifierInterface bundleNotifierInterface];
   [c resume];
   self.notifierConnection = c;
-  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
+  dispatch_async(self.queue, ^{
     [self createConnection];
   });
 }
@@ -84,13 +91,13 @@
 - (void)hashBundleBinariesForEvent:(SNTStoredEvent *)event
                              reply:(SNTBundleHashBlock)reply {
   NSProgress *progress =
-      [NSProgress currentProgress] ? [NSProgress progressWithTotalUnitCount:1] : nil;
+      [NSProgress currentProgress] ? [NSProgress progressWithTotalUnitCount:100] : nil;
 
   NSDate *startTime = [NSDate date];
 
   dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 
-  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
+  dispatch_async(self.queue, ^{
     // Use the highest bundle we can find. Save and reuse the bundle infomation when creating
     // the related binary events.
     SNTFileInfo *b = [[SNTFileInfo alloc] initWithPath:event.fileBundlePath];
@@ -101,19 +108,26 @@
     event.fileBundleVersion = b.bundleVersion;
     event.fileBundleVersionString = b.bundleShortVersionString;
 
-    NSArray *relatedBinaries = [self findRelatedBinaries:event progress:progress];
-    NSString *bundleHash = [self calculateBundleHashFromEvents:relatedBinaries];
+    // For most apps this should be "Contents/MacOS/AppName"
+    if (b.bundle.executablePath.length > b.bundlePath.length) {
+      event.fileBundleExecutableRelPath =
+          [b.bundle.executablePath substringFromIndex:b.bundlePath.length + 1];
+    }
+
+    NSDictionary *relatedEvents = [self findRelatedBinaries:event progress:progress];
+    NSString *bundleHash = [self calculateBundleHashFromSHA256Hashes:relatedEvents.allKeys
+                                                            progress:progress];
+
     NSNumber *ms = [NSNumber numberWithDouble:[startTime timeIntervalSinceNow] * -1000.0];
-    if (bundleHash) LOGD(@"hashed %@ in %@ ms", event.fileBundlePath, ms);
-    reply(bundleHash, relatedBinaries, ms);
+
+    reply(bundleHash, relatedEvents.allValues, ms);
     dispatch_semaphore_signal(sema);
   });
 
   // Master timeout of 10 min. Don't block the calling thread. NSProgress updates will be coming
   // in over this thread.
-  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
+  dispatch_async(self.queue, ^{
     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, 600 * NSEC_PER_SEC))) {
-      LOGD(@"hashBundleBinariesForEvent timeout");
       [progress cancel];
     }
   });
@@ -122,170 +136,144 @@
 #pragma mark Internal Methods
 
 /**
- Find binaries within a bundle given the bundle's event. It will run until a timeout occurs, 
- or until the NSProgress is cancelled. Search is done within the bundle concurrently.
+  Find binaries within a bundle given the bundle's event. It will run until a timeout occurs,
+  or until the NSProgress is cancelled. Search is done within the bundle concurrently.
 
- @param event The SNTStoredEvent to begin searching underneath
- @return An array of SNTStoredEvent's
+  @param event The SNTStoredEvent to begin searching.
+  @return An NSDictionary object with keys of fileSHA256 and values of SNTStoredEvent objects.
+*/
+- (NSDictionary *)findRelatedBinaries:(SNTStoredEvent *)event progress:(NSProgress *)progress {
+  // Find all files and folders within the fileBundlePath
+  NSFileManager *fm = [NSFileManager defaultManager];
+  NSArray *subpaths = [fm subpathsOfDirectoryAtPath:event.fileBundlePath error:NULL];
 
- @note The first stage gathers a set of executables. 60 sec / max thread timeout.
- @note The second stage hashes the executables. 300 sec / max thread timeout.
- */
-- (NSArray *)findRelatedBinaries:(SNTStoredEvent *)event progress:(NSProgress *)progress {
-  // For storing the generated events, with a simple lock for writing.
-  NSMutableArray *relatedEvents = [NSMutableArray array];
-
-  // For storing files to be hashed
-  NSMutableSet<SNTFileInfo *> *fis = [NSMutableSet set];
-
-  // Limit the number of threads that can process files at once to keep CPU usage down.
-  dispatch_semaphore_t sema =
-      dispatch_semaphore_create([[NSProcessInfo processInfo] processorCount] / 2);
-
-  // Group the processing into a single group so we can wait on the whole group after each stage.
-  dispatch_group_t group = dispatch_group_create();
-
-  // Directory enumerator
-  NSDirectoryEnumerator *dirEnum =
-      [[NSFileManager defaultManager] enumeratorAtPath:event.fileBundlePath];
-
-  // Locks for accessing the enumerator and adding file and events between threads.
-  __block pthread_mutex_t enumeratorMutex = PTHREAD_MUTEX_INITIALIZER;
-  __block pthread_mutex_t eventsMutex = PTHREAD_MUTEX_INITIALIZER;
+  // This array is used to store pointers to executable SNTFileInfo objects. There will be one block
+  // dispatched per file in dirEnum. These blocks will write pointers to this array concurrently.
+  // No locks are used since every file has a slot.
+  //
+  // Xcode.app has roughly 500k files, 8bytes per pointer is ~4MB for this array. This size to space
+  // ratio seems appropriate as Xcode.app is in the upper bounds of bundle size.
+  __block void **fis = calloc(subpaths.count, sizeof(void *));
 
   // Counts used as additional progress information in SantaGUI
-  __block uint64_t binaryCount = 0;
-  __block uint64_t sentBinaryCount = 0;
-  __block uint64_t fileCount = 0;
-
-  __block BOOL breakDir = NO;
-
-  // In the first stage iterate over every file in the tree checking if it is a binary. If so add
-  // it to the fis set for the second stage. Hashing the file while iterating over the filesystem
-  // causes performance issues. Do them separately.
-  while (1) {
-    @autoreleasepool {
-      if (breakDir || progress.isCancelled) break;
-
-      // Wait for a processing thread to become available. At this stage we are only reading the
-      // mach_header. If all processing threads are blocking for more than 60 sec bail.
-      if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, 60 * NSEC_PER_SEC))) {
-        LOGD(@"isExecutable processing threads timeout");
-        return nil;
-      }
-
-      dispatch_group_async(group,
-                           dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
-        pthread_mutex_lock(&enumeratorMutex);
-        NSString *file = [dirEnum nextObject];
-        fileCount++;
-        pthread_mutex_unlock(&enumeratorMutex);
-
-        if (!file) {
-          breakDir = YES;
-          dispatch_semaphore_signal(sema);
-          return;
-        }
-
-        if ([dirEnum fileAttributes][NSFileType] != NSFileTypeRegular) {
-          dispatch_semaphore_signal(sema);
-          return;
-        }
-
-        NSString *newFile = [event.fileBundlePath stringByAppendingPathComponent:file];
-        SNTFileInfo *fi = [[SNTFileInfo alloc] initWithPath:newFile];
-        if (!fi.isExecutable) {
-          dispatch_semaphore_signal(sema);
-          return;
-        }
-
-        pthread_mutex_lock(&eventsMutex);
-        [fis addObject:fi];
-        binaryCount++;
-        pthread_mutex_unlock(&eventsMutex);
-
-        dispatch_semaphore_signal(sema);
-      });
-      if (progress && ((fileCount % 500) == 0 || binaryCount > sentBinaryCount)) {
-        sentBinaryCount = binaryCount;
-        [[self.notifierConnection remoteObjectProxy] updateCountsForEvent:event
-                                                              binaryCount:binaryCount
-                                                                fileCount:fileCount];
-      }
-    }
-  }
-
-  if (progress.isCancelled) return nil;
-
-  // Wait for all the processing threads to finish
-  dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+  __block volatile int64_t binaryCount = 0;
+  __block volatile int64_t sentBinaryCount = 0;
 
+  // Account for 80% of the work
   NSProgress *p;
   if (progress) {
-    [progress becomeCurrentWithPendingUnitCount:1];
-    p = [NSProgress progressWithTotalUnitCount:fis.count];
+    [progress becomeCurrentWithPendingUnitCount:80];
+    p = [NSProgress progressWithTotalUnitCount:subpaths.count * 100];
   }
 
-  // In the second stage perform SHA256 hashing on all of the found binaries.
-  for (SNTFileInfo *fi in fis) {
+  // Dispatch a block for every file in dirEnum.
+  dispatch_apply(subpaths.count, self.queue, ^(size_t i) {
     @autoreleasepool {
-      if (progress.isCancelled) break;
+      if (progress.isCancelled) return;
 
-      // Wait for a processing thread to become available. Here we are hashing the entire file.
-      // If all processing threads are blocking for more than 5 min bail.
-      if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, 300 * NSEC_PER_SEC))) {
-        LOGD(@"SHA256 processing threads timeout");
-        return nil;
-      }
+      dispatch_sync(dispatch_get_main_queue(), ^{
+        p.completedUnitCount++;
+        if (progress && ((i % 500) == 0 || binaryCount > sentBinaryCount)) {
+          sentBinaryCount = binaryCount;
+          [[self.notifierConnection remoteObjectProxy] updateCountsForEvent:event
+                                                                binaryCount:binaryCount
+                                                                  fileCount:i
+                                                                hashedCount:0];
+        }
+      });
 
-      dispatch_group_async(group,
-                           dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
-        @autoreleasepool {
-          SNTStoredEvent *se = [[SNTStoredEvent alloc] init];
-          se.filePath = fi.path;
-          se.fileSHA256 = fi.SHA256;
-          se.occurrenceDate = [NSDate distantFuture];
-          se.decision = SNTEventStateBundleBinary;
+      NSString *subpath = subpaths[i];
 
-          se.fileBundlePath = event.fileBundlePath;
-          se.fileBundleID = event.fileBundleID;
-          se.fileBundleName = event.fileBundleName;
-          se.fileBundleVersion = event.fileBundleVersion;
-          se.fileBundleVersionString = event.fileBundleVersionString;
+      NSString *file =
+          [event.fileBundlePath stringByAppendingPathComponent:subpath].stringByStandardizingPath;
+      SNTFileInfo *fi = [[SNTFileInfo alloc] initWithResolvedPath:file error:NULL];
+      if (!fi.isExecutable) return;
 
-          MOLCodesignChecker *cs = [[MOLCodesignChecker alloc] initWithBinaryPath:se.filePath];
-          se.signingChain = cs.certificates;
+      fis[i] = (__bridge_retained void *)fi;
+      OSAtomicIncrement64Barrier(&binaryCount);
+    }
+  });
 
-          pthread_mutex_lock(&eventsMutex);
-          [relatedEvents addObject:se];
-          p.completedUnitCount++;
-          pthread_mutex_unlock(&eventsMutex);
+  [progress resignCurrent];
 
-          dispatch_semaphore_signal(sema);
+  NSMutableArray *fileInfos = [NSMutableArray arrayWithCapacity:binaryCount];
+  for (NSUInteger i = 0; i < subpaths.count; i++) {
+    if (fis[i]) [fileInfos addObject:(__bridge_transfer SNTFileInfo *)fis[i]];
+  }
+
+  free(fis);
+
+  return [self generateEventsFromBinaries:fileInfos blockingEvent:event progress:progress];
+}
+
+- (NSDictionary *)generateEventsFromBinaries:(NSArray *)fis
+                               blockingEvent:(SNTStoredEvent *)event
+                                    progress:(NSProgress *)progress {
+  if (progress.isCancelled) return nil;
+
+  NSMutableDictionary *relatedEvents = [NSMutableDictionary dictionaryWithCapacity:fis.count];
+
+  // Account for 15% of the work
+  NSProgress *p;
+  if (progress) {
+    [progress becomeCurrentWithPendingUnitCount:15];
+    p = [NSProgress progressWithTotalUnitCount:fis.count * 100];
+  }
+
+  dispatch_apply(fis.count, self.queue, ^(size_t i) {
+    @autoreleasepool {
+      if (progress.isCancelled) return;
+
+      SNTFileInfo *fi = fis[i];
+
+      SNTStoredEvent *se = [[SNTStoredEvent alloc] init];
+      se.filePath = fi.path;
+      se.fileSHA256 = fi.SHA256;
+      se.occurrenceDate = [NSDate distantFuture];
+      se.decision = SNTEventStateBundleBinary;
+
+      se.fileBundlePath = event.fileBundlePath;
+      se.fileBundleExecutableRelPath = event.fileBundleExecutableRelPath;
+      se.fileBundleID = event.fileBundleID;
+      se.fileBundleName = event.fileBundleName;
+      se.fileBundleVersion = event.fileBundleVersion;
+      se.fileBundleVersionString = event.fileBundleVersionString;
+
+      MOLCodesignChecker *cs = [[MOLCodesignChecker alloc] initWithBinaryPath:se.filePath];
+      se.signingChain = cs.certificates;
+
+      dispatch_sync(dispatch_get_main_queue(), ^{
+        relatedEvents[se.fileSHA256] = se;
+        p.completedUnitCount++;
+        if (progress) {
+          [[self.notifierConnection remoteObjectProxy] updateCountsForEvent:event
+                                                                binaryCount:fis.count
+                                                                  fileCount:0
+                                                                hashedCount:i];
         }
       });
     }
-  }
+  });
 
-  // Wait for all the processing threads to finish
-  dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+  [progress resignCurrent];
 
-  pthread_mutex_destroy(&enumeratorMutex);
-  pthread_mutex_destroy(&eventsMutex);
-
-  return progress.isCancelled ? nil : relatedEvents;
+  return relatedEvents;
 }
 
-- (NSString *)calculateBundleHashFromEvents:(NSArray<SNTStoredEvent *> *)events {
-  if (!events) return nil;
-  NSMutableArray *eventSHA256Hashes = [NSMutableArray arrayWithCapacity:events.count];
-  for (SNTStoredEvent *event in events) {
-    if (!event.fileSHA256) return nil;
-    [eventSHA256Hashes addObject:event.fileSHA256];
+- (NSString *)calculateBundleHashFromSHA256Hashes:(NSArray *)hashes
+                                         progress:(NSProgress *)progress {
+  if (!hashes.count) return nil;
+
+  // Account for 5% of the work
+  NSProgress *p;
+  if (progress) {
+    [progress becomeCurrentWithPendingUnitCount:5];
+    p = [NSProgress progressWithTotalUnitCount:5 * 100];
   }
 
-  [eventSHA256Hashes sortUsingSelector:@selector(localizedCaseInsensitiveCompare:)];
-  NSString *sha256Hashes = [eventSHA256Hashes componentsJoinedByString:@""];
+  NSMutableArray *sortedHashes = [hashes mutableCopy];
+  [sortedHashes sortUsingSelector:@selector(localizedCaseInsensitiveCompare:)];
+  NSString *sha256Hashes = [sortedHashes componentsJoinedByString:@""];
 
   CC_SHA256_CTX c256;
   CC_SHA256_Init(&c256);
@@ -307,6 +295,8 @@
       digest[24], digest[25], digest[26], digest[27],
       digest[28], digest[29], digest[30], digest[31]];
 
+  p.completedUnitCount++;
+  [progress resignCurrent];
   return sha256;
 }
 

Source/santactl/Commands/SNTCommandBundleInfo.m

@@ -76,4 +76,4 @@ REGISTER_COMMAND_NAME(@"bundleinfo")
   }];
 }
 
-@end
+@end

Source/santactl/Commands/sync/SNTCommandSyncConstants.h

@@ -59,6 +59,7 @@ extern NSString *const kLoggedInUsers;
 extern NSString *const kCurrentSessions;
 extern NSString *const kFileBundleID;
 extern NSString *const kFileBundlePath;
+extern NSString *const kFileBundleExecutableRelPath;
 extern NSString *const kFileBundleName;
 extern NSString *const kFileBundleVersion;
 extern NSString *const kFileBundleShortVersionString;

Source/santactl/Commands/sync/SNTCommandSyncConstants.m

@@ -59,6 +59,7 @@ NSString *const kLoggedInUsers = @"logged_in_users";
 NSString *const kCurrentSessions = @"current_sessions";
 NSString *const kFileBundleID = @"file_bundle_id";
 NSString *const kFileBundlePath = @"file_bundle_path";
+NSString *const kFileBundleExecutableRelPath = @"file_bundle_executable_rel_path";
 NSString *const kFileBundleName = @"file_bundle_name";
 NSString *const kFileBundleVersion = @"file_bundle_version";
 NSString *const kFileBundleShortVersionString = @"file_bundle_version_string";

Source/santactl/Commands/sync/SNTCommandSyncEventUpload.m

@@ -109,6 +109,7 @@
 
   ADDKEY(newEvent, kFileBundleID, event.fileBundleID);
   ADDKEY(newEvent, kFileBundlePath, event.fileBundlePath);
+  ADDKEY(newEvent, kFileBundleExecutableRelPath, event.fileBundleExecutableRelPath);
   ADDKEY(newEvent, kFileBundleName, event.fileBundleName);
   ADDKEY(newEvent, kFileBundleVersion, event.fileBundleVersion);
   ADDKEY(newEvent, kFileBundleShortVersionString, event.fileBundleVersionString);

Source/santactl/Commands/sync/SNTCommandSyncManager.m

@@ -34,6 +34,11 @@
 #import "SNTXPCControlInterface.h"
 #import "SNTXPCSyncdInterface.h"
 
+static NSString *const kFCMActionKey = @"action";
+static NSString *const kFCMFileHashKey = @"file_hash";
+static NSString *const kFCMFileNameKey = @"file_name";
+static NSString *const kFCMTargetHostIDKey = @"target_host_id";
+
 @interface SNTCommandSyncManager () {
   SCNetworkReachabilityRef _reachability;
 }
@@ -199,48 +204,40 @@ static void reachabilityHandler(
 }
 
 - (void)processFCMMessage:(NSDictionary *)FCMmessage withMachineID:(NSString *)machineID {
-  NSData *messageData = [self extractMessageDataFrom:FCMmessage];
+  NSDictionary *message = [self messageFromMessageData:[self messageDataFromFCMmessage:FCMmessage]];
 
-  if (!messageData) {
+  if (!message) {
     LOGD(@"Push notification message is not in the expected format...dropping message");
     return;
   }
 
-  NSError *error;
-  NSDictionary *actionMessage = [NSJSONSerialization JSONObjectWithData:messageData
-                                                                options:0
-                                                                  error:&error];
-  if (!actionMessage) {
-    LOGD(@"Unable to parse push notification message value: %@", error);
+  NSString *action = message[kFCMActionKey];
+  if (!action) {
+    LOGD(@"Push notification message contains no action");
     return;
   }
 
   // Store the file name and hash in a cache. When the rule is actually added, use the cache
   // to build a user notification.
-  NSString *fileHash = actionMessage[@"file_hash"];
-  NSString *fileName = actionMessage[@"file_name"];
+  NSString *fileHash = message[kFCMFileHashKey];
+  NSString *fileName = message[kFCMFileNameKey];
   if (fileName && fileHash) {
     [self.ruleSyncCache setObject:fileName forKey:fileHash];
   }
 
-  NSString *action = actionMessage[@"action"];
-  if (action) {
-    LOGD(@"Push notification action: %@ received", action);
-  } else {
-    LOGD(@"Push notification message contains no action");
-  }
+  LOGD(@"Push notification action: %@ received", action);
 
   if ([action isEqualToString:kFullSync]) {
     [self fullSync];
   } else if ([action isEqualToString:kRuleSync]) {
-    NSString *targetMachineID = actionMessage[@"target_host_id"];
-    if (![targetMachineID isKindOfClass:[NSNull class]] &&
-        [targetMachineID.lowercaseString isEqualToString:machineID.lowercaseString]) {
+    NSString *targetHostID = message[kFCMTargetHostIDKey];
+    if (targetHostID && [targetHostID caseInsensitiveCompare:machineID] == NSOrderedSame) {
+      LOGD(@"Targeted rule_sync for host_id: %@", targetHostID);
       self.targetedRuleSync = YES;
       [self ruleSync];
     } else {
       uint32_t delaySeconds = arc4random_uniform((uint32_t)self.FCMGlobalRuleSyncDeadline);
-      LOGD(@"Staggering rule download: %u second delay", delaySeconds);
+      LOGD(@"Global rule_sync, staggering: %u second delay", delaySeconds);
       [self ruleSyncSecondsFromNow:delaySeconds];
     }
   } else if ([action isEqualToString:kConfigSync]) {
@@ -252,12 +249,33 @@ static void reachabilityHandler(
   }
 }
 
-- (NSData *)extractMessageDataFrom:(NSDictionary *)FCMmessage {
+- (NSData *)messageDataFromFCMmessage:(NSDictionary *)FCMmessage {
   if (![FCMmessage[@"data"] isKindOfClass:[NSDictionary class]]) return nil;
   if (![FCMmessage[@"data"][@"blob"] isKindOfClass:[NSString class]]) return nil;
   return [FCMmessage[@"data"][@"blob"] dataUsingEncoding:NSUTF8StringEncoding];
 }
 
+- (NSDictionary *)messageFromMessageData:(NSData *)messageData {
+  NSError *error;
+  NSDictionary *rawMessage = [NSJSONSerialization JSONObjectWithData:messageData
+                                                             options:0
+                                                               error:&error];
+  if (!rawMessage) {
+    LOGD(@"Unable to parse push notification message data: %@", error);
+    return nil;
+  }
+
+  // Create a new message dropping unexpected values
+  NSArray *allowedKeys = @[ kFCMActionKey, kFCMFileHashKey, kFCMFileNameKey, kFCMTargetHostIDKey ];
+  NSMutableDictionary *message = [NSMutableDictionary dictionaryWithCapacity:allowedKeys.count];
+  for (NSString *key in allowedKeys) {
+    if ([rawMessage[key] isKindOfClass:[NSString class]] && [rawMessage[key] length]) {
+      message[key] = rawMessage[key];
+    }
+  }
+  return message.count ? [message copy] : nil;
+}
+
 #pragma mark sync timer control
 
 - (void)fullSync {

Source/santactl/Commands/sync/SNTCommandSyncRuleDownload.m

@@ -79,7 +79,7 @@
     for (SNTRule *r in self.syncState.downloadedRules) {
       NSString *fileName = [[self.syncState.ruleSyncCache objectForKey:r.shasum] copy];
       [self.syncState.ruleSyncCache removeObjectForKey:r.shasum];
-      if (fileName) {
+      if (fileName.length) {
         NSString *message = [NSString stringWithFormat:@"%@ can now be run", fileName];
         [[self.daemonConn remoteObjectProxy]
             postRuleSyncNotificationWithCustomMessage:message reply:^{}];

Source/santad/SNTEventLog.m

@@ -34,6 +34,8 @@
 // Caches for uid->username and gid->groupname lookups.
 @property NSCache<NSNumber *, NSString *> *userNameMap;
 @property NSCache<NSNumber *, NSString *> *groupNameMap;
+
+@property NSDateFormatter *dateFormatter;
 @end
 
 @implementation SNTEventLog
@@ -49,6 +51,10 @@
     _userNameMap.countLimit = 100;
     _groupNameMap = [[NSCache alloc] init];
     _groupNameMap.countLimit = 100;
+
+    _dateFormatter = [[NSDateFormatter alloc] init];
+    _dateFormatter.dateFormat = @"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
+    _dateFormatter.timeZone = [NSTimeZone timeZoneWithName:@"UTC"];
   }
   return self;
 }
@@ -226,7 +232,14 @@
                         diskProperties[@"DADeviceModel"] ?: @""];
   model = [model stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
 
-  LOGI(@"action=DISKAPPEAR|mount=%@|volume=%@|bsdname=%@|fs=%@|model=%@|serial=%@|bus=%@|dmgpath=%@",
+  double appearance = [diskProperties[@"DAAppearanceTime"] doubleValue];
+  NSString *appearanceDateString =
+      [_dateFormatter stringFromDate:[NSDate dateWithTimeIntervalSinceReferenceDate:appearance]];
+
+  NSString *log =
+      @"action=DISKAPPEAR|mount=%@|volume=%@|bsdname=%@|fs=%@|"
+      @"model=%@|serial=%@|bus=%@|dmgpath=%@|appearance=%@";
+  LOGI(log,
        [diskProperties[@"DAVolumePath"] path] ?: @"",
        diskProperties[@"DAVolumeName"] ?: @"",
        diskProperties[@"DAMediaBSDName"] ?: @"",
@@ -234,7 +247,8 @@
        model ?: @"",
        serial,
        diskProperties[@"DADeviceProtocol"] ?: @"",
-       dmgPath);
+       dmgPath,
+       appearanceDateString);
 }
 
 - (void)logDiskDisappeared:(NSDictionary *)diskProperties {