Fixing time flow control +semver: minor +tag +release +publish

* Fixing time interval flow control logic
* Switching time flow control logic to AzureSearcher class to make for cleaner code
* Adding GIF demo to README
* Adding PowershellGallery download badge

AzureHunter.build.ps1

@@ -8,7 +8,7 @@ $script:RelativeSourcePathName = 'Source' # Relative name of the directory where
 $script:SourcePath = Join-Path $BuildRoot $RelativeSourcePathName # Where to source our module manifest and files from
 $script:BuildOutputFolder = Join-Path $BuildRoot Output # Folder where the output of all build & release tasks will be placed
 $script:ReleaseDir = Join-Path $script:BuildOutputFolder Release # Folder where the release package or zip will be placed
-$script:BuildDestinationFolder = Join-Path $BuildOutputFolder $ModuleName # Folder inside $BuildOutputFolder where we will place our built artefacts
+$script:BuildDestinationFolder = Join-Path $BuildOutputFolder $ModuleName # Folder inside $BuildOutputFolder where we will place our built artefacts or final Module
 $script:ExcludedDirs = ( 'private', 'public', 'classes', 'enums' )
 
 # Let's provide stdout output with information on relevant environment context
@@ -18,7 +18,7 @@ Task ContextAwareness {
     Write-Build Yellow "[AzureHunter][Build] Full Source Path: $SourcePath"
     Write-Build Yellow "[AzureHunter][Build] CI/CD Output Folder: $BuildOutputFolder"
     Write-Build Yellow "[AzureHunter][Build] BuildRoot Directory is $BuildRoot"
-    Write-Build Yellow "[AzureHunter][Build] Build Directory is $BuildDestinationFolder"
+    Write-Build Yellow "[AzureHunter][Build] Module Build Directory inside BuildRoot is $BuildDestinationFolder"
     Write-Build Yellow "[AzureHunter][Build] Release Directory is $ReleaseDir"
     Write-Output "`n"
 
@@ -29,6 +29,8 @@ Task PreCleanBuildFolder {
     Write-Build Yellow "`n[AzureHunter][Build] Deleting Folder $BuildOutputFolder"
     $null = Remove-Item $BuildOutputFolder -Recurse -ErrorAction Ignore
     $null = New-Item -Type Directory -Path $BuildDestinationFolder
+    # Unregister repository
+    Unregister-PSRepository AzureHunterDemoRepo -Verbose -ErrorAction SilentlyContinue
 }
 
 # This task will compile all of the PS1 files into a cohesive PSM1 module file
@@ -41,7 +43,7 @@ Task CompilePSM {
         $BuildParams = @{}
         $GitVersion = gitversion | ConvertFrom-Json | Select-Object -Expand SemVer
         $BuildParams['SemVer'] = $GitVersion
-        Write-Build Yellow "`n[AzureHunter][Build] Build Version according to GitVersion: $($BuildParams["SemVer"])"
+        Write-Build Yellow "`n[AzureHunter][Build] Build Version according to GitVersion: $($BuildParams["SemVer"]). THIS IS THE VERSION THAT WILL BE PUBLISHED TO POWERSHELLGALLERY"
     }
     catch {
         Write-Host $Error[0]
@@ -155,6 +157,25 @@ Task PublishUnitTestsCoverage {
     Publish-Coverage -Coverage $Coverage
 }
 
+Task PublishLocalTestPackage {
+    # This task will register a local folder as a destination for a NuGet package and publish
+    # The current build to it so that it can be tested locally
+
+    $RepositoryName = "AzureHunterDemoRepo"
+    $RepositoryPath = Join-Path $script:BuildOutputFolder $RepositoryName
+    Write-Build Green "`n[AzureHunter][Test] Nugget Test Repo Destination Path $RepositoryPath"
+
+    if(!(Test-Path $RepositoryPath)) {
+        New-Item -Type Directory -Path $RepositoryPath
+    }
+    Register-PSRepository -Name $RepositoryName -SourceLocation $RepositoryPath -PublishLocation $RepositoryPath -InstallationPolicy Trusted
+    Publish-Module -Path $script:BuildDestinationFolder -Repository $RepositoryName -NuGetApiKey "TEST"
+
+    # To Test the package simply uninstall from previous locations and re-install from the local repository
+    # Install-Module AzureHunter -Repository AzureHunterDemoRepo -Scope CurrentUser
+    # Get-Module AzureHunter | fl
+}
+
 # This task will update the source Manifest File with the newly built one
 Task UpdateSource {
     Copy-Item $Global:CompileResult.Path -Destination "$SourcePath\$ModuleName.psd1"
@@ -184,4 +205,5 @@ Task PublishPackage {
 Task Default ContextAwareness, PreCleanBuildFolder, CompilePSM, CopyDependenciesToOutput, ZipOutput
 Task UpdateSourceManifest ContextAwareness, UpdateSource
 Task PublishModule ContextAwareness, PublishPackage
-Task TagLatestGitCommit GitTag
+Task TestBuildAndPublishModule Default, PublishLocalTestPackage
+Task TagLatestGitCommit GitTag

README.MD

@@ -7,6 +7,9 @@ A Powershell module to run threat hunting playbooks on data from Azure and O365
 [![PS Gallery](https://img.shields.io/badge/install-PS%20Gallery-blue.svg)](https://www.powershellgallery.com/packages/AzureHunter/)
 [![Coverage Status](https://coveralls.io/repos/github/darkquasar/AzureHunter/badge.svg?branch=master)](https://coveralls.io/github/darkquasar/AzureHunter?branch=master)
 [![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://AzureHunter.readthedocs.io/en/latest/?badge=latest)
+![PowerShell Gallery](https://img.shields.io/powershellgallery/dt/AzureHunter)
+
+![azurehunter-demo-gif](.\media\azurehunter-gif-demo.gif)
 
 ## Getting Started
 

source/classes/02.AzureHunter.Logger.ps1

@@ -76,10 +76,19 @@ class Logger {
 
         # Should we log an Error?
         if ($null -ne $LogErrorMessage) {
-            Write-Host $Error[0]
-            Write-Host $Error
             # Grab latest error namespace
-            $ErrorNameSpace = $Error[0].Exception.GetType().FullName
+            try {
+                $ErrorNameSpace = $Error[0].Exception.GetType().FullName
+            }
+            catch {
+                try {
+                    $ErrorNameSpace = $Error.Exception.GetType().FullName
+                }
+                catch {
+                    $ErrorNameSpace = "Undetermined"
+                }
+            }
+
             # Add Error specific fields
             $LogRecord.Add("error_name_space", $ErrorNameSpace)
             $LogRecord.Add("error_script_line", $LogErrorMessage.InvocationInfo.ScriptLineNumber)
@@ -125,7 +134,7 @@ class Logger {
                 $this.BackgroundColor = "Green"
             }
             "Debug" {
-                $this.MessageColor = "Green"
+                $this.MessageColor = "Black"
                 $this.BackgroundColor = "DarkCyan"
             }
 

source/classes/03.AzureHunter.TimeStamp.ps1

@@ -5,7 +5,8 @@ class TimeStamp {
     # Public Properties
     [float] $Interval
     [float] $IntervalInMinutes
-    [bool] $IntervalAdjusted
+    [float] $UserDefinedInitialTimeInterval # this is the value passed by the user when invoking Search-AzureCloudUnifiedLog
+    [bool] $InitialIntervalAdjusted
     [System.Globalization.CultureInfo] $Culture
     [DateTime] $StartTime
     [DateTime] $EndTime
@@ -17,10 +18,11 @@ class TimeStamp {
     [DateTime] $EndTimeSliceUTC
 
     # Default, Overloaded Constructor
-    TimeStamp([String] $StartTime, [String] $EndTime) {
+    TimeStamp([String] $StartTime, [String] $EndTime, [float] $UserDefinedInitialTimeInterval) {
         $this.Culture = New-Object System.Globalization.CultureInfo("en-AU")
         $this.StartTime = $this.ParseDateString($StartTime)
         $this.EndTime = $this.ParseDateString($EndTime)
+        $this.UserDefinedInitialTimeInterval = $UserDefinedInitialTimeInterval
         $this.UpdateUTCTimestamp()
     }
 

source/classes/04.AzureHunter.AzureSearcher.ps1

@@ -11,6 +11,8 @@ class AzureSearcher {
     [DateTime] $EndTimeUTC
     [String] $SessionId
     [TimeStamp] $TimeSlicer
+    [int] $ResultSizeUpperThreshold
+    [int] $ResultCountEstimate = 0
 
     [AzureSearcher] SetOperations([String[]] $Operations) {
         $this.Operations = $Operations
@@ -33,10 +35,11 @@ class AzureSearcher {
     }
 
     # Default, Overloaded Constructor
-    AzureSearcher([TimeStamp] $TimeSlicer) {
+    AzureSearcher([TimeStamp] $TimeSlicer, [int] $ResultSizeUpperThreshold) {
         $this.TimeSlicer = $TimeSlicer
         $this.StartTimeUTC = $TimeSlicer.StartTimeSliceUTC
         $this.EndTimeUTC = $TimeSlicer.EndTimeSliceUTC
+        $this.ResultSizeUpperThreshold = $ResultSizeUpperThreshold
     }
 
     [Array] SearchAzureAuditLog([String] $SessionId) {
@@ -104,4 +107,146 @@ class AzureSearcher {
             throw $_
         }
     }
+
+    AdjustTimeInterval([String] $AdjustmentMode, [String] $AzureLogSearchSessionName, [Int] $ResultCount) {
+
+        # AdjustmentType: whether we should adjust time interval by increasing it or reducing it
+        # AdjustmentMode: whether we should adjust time interval based on proportion or percentage
+
+        # Run initial check of actions to perform
+        $NeedToFetchLogs = $false
+        if($ResultCount) {
+            $NeedToFetchLogs = $false
+            $this.ResultCountEstimate = $ResultCount
+        }
+        else {
+            $NeedToFetchLogs = $true
+        }
+
+        # **** START: TIME WINDOW FLOW CONTROL ROUTINE **** #
+        # ************************************************* #
+        # This routine performs a series of checks to determine whether the time window 
+        # used for log extraction needs to be adjusted or not, in order to extract the 
+        # highest density of logs within a specified time interval
+
+        # Only run this block if SkipAutomaticTimeWindowReduction is not set.
+        # Determine initial optimal time interval (likely to be less than 30 min anyway) or whenever required by downstream log extractors
+        $TimeWindowAdjustmentNumberOfAttempts = 1
+        $ToleranceBeforeIncrementingTimeSlice = 3 # This controls how many cycles we will run before increasing the TimeSlice after getting ZERO results (I said zero, not null or empty)
+        $ToleranceCounter = 1
+
+        while(($TimeWindowAdjustmentNumberOfAttempts -le 3) -and ($NeedToFetchLogs -eq $true)) {
+
+
+
+            # Run initial query to estimate results and adjust time intervals
+            try {
+                $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Initial TimeSlice in local time: [StartDate] $($this.TimeSlicer.StartTimeSlice.ToString($this.TimeSlicer.Culture)) - [EndDate] $($this.TimeSlicer.EndTimeSlice.ToString($this.TimeSlicer.Culture))", "INFO", $null, $null)
+                $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Extracting data from Azure to estimate initial result size", "INFO", $null, $null)
+
+                $Results = $this.SearchAzureAuditLog($AzureLogSearchSessionName)
+
+
+            }
+            catch [System.Management.Automation.RemoteException] {
+                $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Failed to query Azure API during initial ResultCountEstimate. Please check passed parameters and Azure API error", "ERROR", $null, $_)
+                break
+            }
+            catch {
+                Write-Host "ERROR ON: $_"
+                if($TimeWindowAdjustmentNumberOfAttempts -lt 3) {
+                    $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Failed to query Azure API during initial ResultCountEstimate: Attempt $TimeWindowAdjustmentNumberOfAttempts of 3. Trying again", "ERROR", $null, $_)
+                    $TimeWindowAdjustmentNumberOfAttempts++
+                    continue
+                }
+                else {
+                    $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Failed to query Azure API during initial ResultCountEstimate: Attempt $TimeWindowAdjustmentNumberOfAttempts of 3. Exiting...", "ERROR", $null, $null)
+                    break
+                }
+            }
+
+            # Now check whether we got ANY RESULTS BACK AT ALL, if not, then there are no results for this particular timewindow. We need to increase timewindow and start again.
+            try {
+                $this.ResultCountEstimate = $Results[0].ResultCount
+                $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Initial Result Size estimate: $($this.ResultCountEstimate)", "INFO", $null, $null)
+            }
+            catch {
+                $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] No results were returned with the current parameters within the designated time window. Increasing timeslice.", "LOW", $null, $null)
+                $this.TimeSlicer.IncrementTimeSlice($this.TimeSlicer.UserDefinedInitialTimeInterval)
+                continue
+            }
+
+            # If we get to this point then it means we have at least received SOME results back.
+            # Check if the ResultEstimate is within expected limits.
+            # If it is, then break from Time Window Flow Control routine and proceed to log extraction process with new timeslice
+            if($this.ResultCountEstimate -le $this.ResultSizeUpperThreshold) {
+
+                if($this.ResultCountEstimate -eq 0) {
+
+                    if($ToleranceCounter -le $ToleranceBeforeIncrementingTimeSlice) {
+                        # Probably an error, we need to do it again
+                        $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Query to Azure API during initial ResultCountEstimate returned ZERO results. This could be an API error. Attempting to retrieve results again BEFORE INCREMENTING TIMESLICE: Attempt $ToleranceCounter of $ToleranceBeforeIncrementingTimeSlice.", "LOW", $null, $null)
+                        $ToleranceCounter++
+                        continue
+                    }
+                    else {
+                        $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Query to Azure API during initial ResultCountEstimate returned ZERO results after too many attempts. There are no logs within current time interval. Increasing it by user defined $($this.TimeSlicer.UserDefinedInitialTimeInterval).", "ERROR", $null, $null)
+                        $this.TimeSlicer.IncrementTimeSlice($this.TimeSlicer.UserDefinedInitialTimeInterval)
+                        # Reset $ToleranceCounter
+                        $ToleranceCounter = 1
+                    }
+
+                }
+                else {
+                    # Results are not ZERO and are within the expected Threshold. Great news!
+
+                    $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Result Size estimate of $($this.ResultCountEstimate) in current time interval within expected threshold of $($this.ResultSizeUpperThreshold). No need to perform further time adjustments. Proceeding...", "INFO", $null, $null)
+
+                    # Set control flags
+                    $this.TimeSlicer.InitialIntervalAdjusted = $true
+                    # Results within appettite, no need to adjust interval again
+                    return
+                }
+
+
+            }
+            else {
+                break # break and go into TimeAdjustment routine below
+            }
+        }
+
+        # This OptimalTimeIntervalCheck helps shorten the time it takes to arrive to a proper time window within the expected ResultSize window
+        # Perform optimal time interval calculation via proportional estimation
+        if($AdjustmentMode -eq "ProportionalAdjustment") {
+
+            $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Size of results is too big. Estimating Optimal Hourly Time Interval...", "DEBUG", $null, $null)
+            $OptimalTimeSlice = ($this.ResultSizeUpperThreshold * $this.TimeSlicer.UserDefinedInitialTimeInterval) / $this.ResultCountEstimate
+            $OptimalTimeSlice = [math]::Round($OptimalTimeSlice, 3)
+            $IntervalInMinutes = $OptimalTimeSlice * 60
+            $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Estimated Optimal Hourly Time Interval: $OptimalTimeSlice ($IntervalInMinutes minutes). Reducing interval to this value...", "DEBUG", $null, $null)
+
+            $this.TimeSlicer.UserDefinedInitialTimeInterval = $OptimalTimeSlice
+            $this.TimeSlicer.Reset()
+            $this.TimeSlicer.IncrementTimeSlice($OptimalTimeSlice)
+            $this.TimeSlicer.InitialIntervalAdjusted = $true
+
+            return
+        }
+        # Perform time interval adjustment based on IntevalReductionRate
+        # if requested by downstream data processors
+        elseif($AdjustmentMode -eq "PercentageAdjustment") {
+            $TimeIntervalReductionRate = 0.2
+
+            $AdjustedHourlyTimeInterval = $this.TimeSlicer.UserDefinedInitialTimeInterval - ($this.TimeSlicer.UserDefinedInitialTimeInterval * $TimeIntervalReductionRate)
+            $AdjustedHourlyTimeInterval = [math]::Round($AdjustedHourlyTimeInterval, 3)
+            $IntervalInMinutes = $AdjustedHourlyTimeInterval * 60
+            $Global:Logger.LogMessage("[INTERVAL FLOW CONTROL] Size of results is too big. Reducing Hourly Time Interval by $TimeIntervalReductionRate to $AdjustedHourlyTimeInterval hours ($IntervalInMinutes minutes)", "INFO", $null, $null)
+
+            $this.TimeSlicer.UserDefinedInitialTimeInterval = $AdjustedHourlyTimeInterval
+            $this.TimeSlicer.Reset()
+            $this.TimeSlicer.IncrementTimeSlice($AdjustedHourlyTimeInterval)
+
+            return
+        }
+    }
 }

source/public/Invoke-HuntAzureAuditLogs.ps1

@@ -6,10 +6,6 @@
     Description: This module contains some utilities to search through Azure and O365 unified audit log.
 #>
 
-using namespace AzureHunter.AzureSearcher
-using namespace AzureHunter.Logger
-using namespace AzureHunter.TimeStamp
-
 Function Invoke-HuntAzureAuditLogs {
     <#
     .SYNOPSIS
@@ -68,16 +64,34 @@ Function Invoke-HuntAzureAuditLogs {
 
         # Determine path to Playbooks folder
         # This is required to pre-load the Base Playbook "AzHunterBase" to do initial sanitization of logs
-        $ScriptPath = [System.IO.DirectoryInfo]::new($pwd)
-        $SourceFolderPresent = Get-ChildItem -Path $ScriptPath.FullName -Directory -Filter "source"
-        if($SourceFolderPresent){
-            $Script:PlaybooksPath = Join-Path $ScriptPath "source\playbooks"
+        if ($PSScriptRoot) {
+            $ScriptPath = [System.IO.DirectoryInfo]::new($PSScriptRoot)
+            if($ScriptPath.FullName -match "AzureHunter\\source"){
+                $ScriptPath = $ScriptPath.Parent
+                $Script:PlaybooksPath = Join-Path $ScriptPath.FullName "playbooks"
+            }
+            else {
+                $Script:PlaybooksPath = Join-Path $ScriptPath.FullName "playbooks"
+            }
         }
         else {
-            $Script:PlaybooksPath = Join-Path $ScriptPath playbooks
+            $ScriptPath = [System.IO.DirectoryInfo]::new($pwd)
+            $PlaybooksFolderPresent = Get-ChildItem -Path $ScriptPath.FullName -Directory -Filter "Playbooks"
+            if($PlaybooksFolderPresent){
+                $Script:PlaybooksPath = Join-Path $ScriptPath "playbooks"
+            }
+            else {
+                throw "Could not find Playbooks folder"
+            }
         }
+
         # Load Base Playbook
-        . "$Script:PlaybooksPath\AzHunter.Playbook.Base.ps1"
+        try {
+            . "$Script:PlaybooksPath\AzHunter.Playbook.Base.ps1"
+        }
+        catch {
+            $Logger.LogMessage("Could not load AzHunter.Playbook.Base", "ERROR", $null, $_)
+        }
 
         # Grab List of All Playbook File Paths
         [System.Collections.ArrayList]$PlaybookFileList = @()
@@ -114,12 +128,17 @@ Function Invoke-HuntAzureAuditLogs {
                 $PlaybookBaseName = $_.BaseName
                 $Logger.LogMessage("Evaluating Playbook $PlaybookBaseName", "INFO", $null, $null)
                 if($PlaybookBaseName -eq $Playbook) {
-                    . $_.FullName
-                    if($PassThru) {
-                        $ReturnRecords = Start-AzHunterPlaybook -Records $BasePlaybookRecords.AzureHuntersRecordsArray -PassThru
+                    try {
+                        . $_.FullName
+                        if($PassThru) {
+                            $ReturnRecords = Start-AzHunterPlaybook -Records $BasePlaybookRecords.AzureHuntersRecordsArray -PassThru
+                        }
+                        else {
+                            Start-AzHunterPlaybook -Records $BasePlaybookRecords.AzureHuntersRecordsArray
+                        }
                     }
-                    else {
-                        Start-AzHunterPlaybook -Records $BasePlaybookRecords.AzureHuntersRecordsArray
+                    catch {
+                        $Logger.LogMessage("Could not load Playbook $Playbook", "ERROR", $null, $_)
                     }
                 }
             }