Runner reliability update (#174)

* spawn process * start-process * corrections * typo fix * asJob * backup restart task * trim newline * trim newline * trim newline * no additional process * cleanup * cleanup
redcanaryco · Dec 8, 2023 · 14b7721 · 14b7721
1 parent 0f44a0b
commit 14b7721
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 25 deletions.
diff --git a/Private/Write-KeyValue.ps1 b/Private/Write-KeyValue.ps1
@@ -1,5 +1,5 @@
 function Write-KeyValue ($key, $value) {
-    Write-Host -ForegroundColor  Cyan -NoNewline $key
+    Write-Host -ForegroundColor Cyan -NoNewline $key
     $split = $value -split "(#{[a-z-_A-Z]*})"
     foreach ($s in $split){
         if($s -match "(#{[a-z-_A-Z]*})"){

diff --git a/Public/Invoke-AtomicRunner.ps1 b/Public/Invoke-AtomicRunner.ps1
@@ -114,6 +114,7 @@ function Invoke-AtomicRunner {
                         # add retry loop to avoid this occassional error "The verification of the MSA failed with error 1355"
                         Invoke-Command -ComputerName '127.0.0.1' -ConfigurationName 'RenameRunnerEndpoint' -ScriptBlock { Rename-Computer -NewName $Using:newHostName -Force -Restart }
                         Start-Sleep 120; $count = $count + 1
+                        LogRunnerMsg "Retrying computer rename $count"
                         if ($count -gt 15) { $retry = $false }
                     }
                 }
@@ -150,7 +151,7 @@ function Invoke-AtomicRunner {
             if ($sleeptime -lt 120) { $sleeptime = 120 } # minimum 2 minute sleep time
             return $sleeptime
         }
-
+        
         # Convert OtherArgs to hashtable so we can pass it through to the call to Invoke-AtomicTest
         $htvars = @{}
         if ($OtherArgs) {
@@ -217,10 +218,10 @@ function Invoke-AtomicRunner {
             }
 
             if ($null -ne $tr) {
+                # run the atomic test and exit
                 Invoke-AtomicTestFromScheduleRow $tr
-                Write-Host -Fore cyan "Sleeping for $SleepTillCleanup seconds before cleaning up"; Start-Sleep -Seconds $SleepTillCleanup
-
                 # Cleanup after running test
+                Write-Host -Fore cyan "Sleeping for $SleepTillCleanup seconds before cleaning up for $($tr.Technique) $($tr.auto_generated_guid) "; Start-Sleep -Seconds $SleepTillCleanup
                 Invoke-AtomicTestFromScheduleRow $tr $true
             }
             else {

diff --git a/Public/Invoke-KickoffAtomicRunner.ps1 b/Public/Invoke-KickoffAtomicRunner.ps1
@@ -2,9 +2,10 @@ function Invoke-KickoffAtomicRunner {
 
     #log rotation function
     function Rotate-Log {
-        Param ($log, $max_filesize, $max_age)
+        Param ($logPath, $max_filesize, $max_age)
         $datetime = Get-Date -uformat "%Y-%m-%d-%H%M"
 
+        $log = Get-Item $logPath
         if ($log.Length / 1MB -ge $max_filesize) { 
             Write-Host "file named $($log.name) is bigger than $max_filesize MB"
             $newname = "$($log.Name)_${datetime}.arclog"
@@ -22,34 +23,21 @@ function Invoke-KickoffAtomicRunner {
         }
     }
 
-    #Check if logfiles exist. If not create them.
+    #Create log files as needed
     $all_log_file = Join-Path $artConfig.atomicLogsPath "all-out-$($artConfig.basehostname).txt"
-    if ($False -eq (Test-Path $all_log_file)) {
-        New-Item $all_log_file -ItemType File -Force
-    }
-    if ($False -eq (Test-Path $artConfig.logFile)) {
-        New-Item $artConfig.logFile -ItemType File -Force
-    }
-
+    New-Item $all_log_file -ItemType file -ErrorAction Ignore 
+    New-Item $artConfig.logFile -ItemType File -ErrorAction Ignore 
 
     #Rotate logs based on FileSize and Date max_filesize
     $max_filesize = 200 #in MB
     $max_file_age = 30 #in days
-    $log = get-item $all_log_file
-    Rotate-Log $log $max_filesize $max_file_age
-    $log = get-item $artConfig.logFile
-    Rotate-Log $log $max_filesize $max_file_age #no need to repeat this. Can reduce further.
+    Rotate-Log $all_log_file $max_filesize $max_file_age
+    Rotate-Log $artConfig.logFile $max_filesize $max_file_age #no need to repeat this. Can reduce further.
 
     # Optional additional delay before starting
     Start-Sleep $artConfig.kickOffDelay.TotalSeconds
 
-    # Invoke the Runner Script
-    if ($artConfig.debug) {
-        Invoke-AtomicRunner *>> $all_log_file
-    }
-    else {
-        Invoke-AtomicRunner
-    }
+    if ($artConfig.debug) { Invoke-AtomicRunner  *>> $all_log_file } else { Invoke-AtomicRunner  }
 }
 
 function LogRunnerMsg ($message) {

diff --git a/Public/Invoke-SetupAtomicRunner.ps1 b/Public/Invoke-SetupAtomicRunner.ps1
@@ -56,7 +56,9 @@ function Invoke-SetupAtomicRunner {
         # local security policy --> Local Policies --> Security Options --> Network access: Do not allow storage of passwords and credentials for network authentication must be disabled
         $taskName = "KickOff-AtomicRunner"
         Unregister-ScheduledTask $taskName -confirm:$false -ErrorAction Ignore
-        $taskAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-exec bypass -Command Invoke-KickoffAtomicRunner"
+        # Windows scheduled task includes a 20 minutes sleep then restart if the call to Invoke-KickoffAtomicRunner fails
+        # this occurs occassional when Windows has issues logging into the runner user's account and logs in as a TEMP user
+        $taskAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-exec bypass -Command Invoke-KickoffAtomicRunner; Start-Sleep 1200; Restart-Computer -Force"
         $taskPrincipal = New-ScheduledTaskPrincipal -UserId $artConfig.user
         $delays = @(1, 2, 4, 8, 16, 32, 64) # using multiple triggers as a retry mechanism because the built-in retry mechanism doesn't work when the computer renaming causes AD replication delays
         $triggers = @()