From e0b412663a8d05d76d4d4efe4333ef3105215833 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 4 May 2024 16:02:27 +1000 Subject: [PATCH 1/5] Keep better track of rejected rules. This PR writes rejected rules into a file and checks against this file each compile. This allows us to review for new rules that fail using git diff and supporess error messages for rules which are already known to be broken. This PR also returns a failed error code when an unhandled error occurs which allows it to be effective in CI now. Additionally this PR also started adding some VQL based sigma rules for example checking for presence of files or registry keys. --- Makefile | 2 +- config/windows_hayabusa_rules.yaml | 32 +++- go.mod | 6 +- go.sum | 4 +- rejected/windows_hayabusa_rejects.json | 204 +++++++++++++++++++++++++ rules/vql/DOTNET_STARTUP_HOOKS.yml | 32 ++++ rules/vql/DSRMBackdoor.yml | 29 ++++ rules/vql/OfficeAI.yaml | 36 +++++ rules/vql/rclone.yml | 31 ++++ src/artifact.go | 6 + src/compile.go | 60 ++++++-- src/logsources.go | 94 ++++++++++-- src/rejects.go | 35 +++++ src/utils.go | 5 + 14 files changed, 544 insertions(+), 32 deletions(-) create mode 100644 rejected/windows_hayabusa_rejects.json create mode 100644 rules/vql/DOTNET_STARTUP_HOOKS.yml create mode 100644 rules/vql/DSRMBackdoor.yml create mode 100644 rules/vql/OfficeAI.yaml create mode 100644 rules/vql/rclone.yml create mode 100644 src/rejects.go diff --git a/Makefile b/Makefile index 527e41a..80e5cf3 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ compile: compileThirdParty compileCurated compileThirdParty: compileHayabusa compileHayabusaMonitoring compileChopChopGo compileHayabusa: - ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml + ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects debugHayabusa: dlv debug ./src -- compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml diff --git a/config/windows_hayabusa_rules.yaml b/config/windows_hayabusa_rules.yaml index 4ce8710..65182e0 100644 --- a/config/windows_hayabusa_rules.yaml +++ b/config/windows_hayabusa_rules.yaml @@ -1,7 +1,7 @@ Preamble: | name: Windows.Hayabusa.Rules description: | - This artifact compiles the Hayabusa rulese into a Velociraptor + This artifact compiles the Hayabusa ruleset into a Velociraptor artifact using the internal Velociraptor sigma engine. To read more about this artifact see @@ -77,6 +77,16 @@ Preamble: | `%%1937`= 'ELEVATED_TOKEN', `%%1938`= 'LIMITED_TOKEN') + // Helpers for VQL rules + LET FetchKeyValues(OSPath) = to_dict(item={ + SELECT Name AS _key, Data.value AS _value + FROM glob(globs="*", accessor="registry", root=OSPath) + }) + + LET GetValue(OSPath) = stat(filename=OSPath, accessor="registry").Data.value + LET Hostname <= dict(H={ SELECT Hostname FROM info()}).H[0].Hostname + + FieldMappings: MandatoryLabel: | x=>get(item=MandatoryLabelLookup, member=x.EventData.MandatoryLabel || "-") @@ -176,6 +186,7 @@ FieldMappings: LogonID: "x=>x.EventData.LogonID" LogonProcessName: "x=>x.EventData.LogonProcessName" LogonType: "x=>x.EventData.LogonType" + Logon_Type: "x=>x.EventData.LogonType" Logon_Account: "x=>x.EventData.Logon_Account" MachineName: "x=>x.EventData.MachineName" MemberName: "x=>x.EventData.MemberName" @@ -365,6 +376,8 @@ FieldMappings: # antivirus/windows/windefend Feature_Name: "x=>x.EventData.`Feature Name`" + EventData: "x=>x.EventData" + DefaultDetails: Query: | x=>get(item=DefaultDetails, @@ -921,6 +934,16 @@ Sources: - Channel - EventID + vql/windows/*: + query: | + SELECT dict( + Computer=Hostname, + Channel="VQL Evaluation", + TimeCreated=dict(SystemTime=now()) + ) AS System, + dict() AS EventData + FROM scope() + QueryTemplate: | LET Rules <= SigmaRules || gunzip(string=base64decode(string="{{.Base64CompressedRules}}")) LET FieldMapping <= parse_json(data=gunzip(string=base64decode(string="{{.Base64FieldMapping}}"))) @@ -938,7 +961,7 @@ QueryTemplate: | dict(System=System, EventData=X.EventData || X.UserData, Message=X.Message) AS _Event, - _Match + _Match, * FROM sigma( rules=split(string= Rules, sep_string="\n---\n"), log_sources= LogSources, debug=Debug, @@ -977,6 +1000,10 @@ BadFieldMappings: # hayabusa/sigma/builtin/security/win_security_rdp_reverse_tunnel.yml - FilterOrigin +# We do not currently support rules with Timeframe in them so we +# silently drop these. +- Timeframe detections not supported + # Many rules are broken and have bad log source section. The following # list suppresses these warnings (but the rules are still rejected) @@ -997,3 +1024,4 @@ RuleDirectories: - hayabusa/hayabusa/sysmon/ - hayabusa/sigma/builtin/ - hayabusa/sigma/sysmon/ + - rules/vql/ diff --git a/go.mod b/go.mod index 386bc17..b2b599e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/alecthomas/kingpin/v2 v2.3.2 github.com/bradleyjkemp/sigma-go v0.6.4 github.com/davecgh/go-spew v1.1.1 + github.com/sebdah/goldie v1.0.0 github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a gopkg.in/yaml.v3 v3.0.1 @@ -17,7 +18,10 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/sebdah/goldie v1.0.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) + +replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed + +//replace github.com/bradleyjkemp/sigma-go => ../sigma-go diff --git a/go.sum b/go.sum index b7bf820..e1ff0f0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed h1:zqhuWeg6oqO3jNabjKJaGO7DreiGhbVfeyqleICMAZk= +github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4= github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= @@ -8,8 +10,6 @@ github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1p github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs= -github.com/bradleyjkemp/sigma-go v0.6.4 h1:J6Sqwbgv7wsEuP7xbsG8dvTrTc9lhkf5BvYF+gO9vzc= -github.com/bradleyjkemp/sigma-go v0.6.4/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/rejected/windows_hayabusa_rejects.json b/rejected/windows_hayabusa_rejects.json new file mode 100644 index 0000000..4beb6ea --- /dev/null +++ b/rejected/windows_hayabusa_rejects.json @@ -0,0 +1,204 @@ +{ + "Rejects": [ + { + "Path": "hayabusa/hayabusa/builtin/Partition_Diagnostic/PartitionDiagnostic_1006_Info_DeviceConn.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/Security/LogonLogoff/Logon/Sec_4624_Med_Logon-Type9-NewInteractive_PossibleTokenImpersonation.yml", + "Error": "Invalid modifier equalsfield" + }, + { + "Path": "hayabusa/hayabusa/builtin/Security/LogonLogoff/Logon/Sec_4624_Med_Logon-Type9-NewInteractive_PossibleTokenImpersonation.yml", + "Error": "Invalid modifier equalsfield" + }, + { + "Path": "hayabusa/hayabusa/builtin/Security/LogonLogoff/Logon/Sec_4625_Med_LogonFail_UsrGuessing_Cnt.yml", + "Error": "yaml: unmarshal errors:\n line 27: cannot unmarshal !!str `https:/...` into []string" + }, + { + "Path": "hayabusa/hayabusa/builtin/Security/LogonLogoff/Logon/Sec_4625_Med_LogonFail_WrongPW_PW-Guessing_Cnt.yml", + "Error": "yaml: unmarshal errors:\n line 29: cannot unmarshal !!str `https:/...` into []string" + }, + { + "Path": "hayabusa/hayabusa/builtin/Security/LogonLogoff/Logon/Sec_4648_Med_ExplicitLogon_PW-Spray_Cnt.yml", + "Error": "yaml: unmarshal errors:\n line 27: cannot unmarshal !!str `https:/...` into []string" + }, + { + "Path": "hayabusa/hayabusa/builtin/TerminalServices-RDPClient_Op/RDP-Client_1024_Info_ConnAttempt.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/TerminalServices-RDPClient_Op/RDP-Client_1102_Info_ConnAttempt.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/TerminalServices-RemoteConnectionManager_Op/RemoteConnManger_1149_Info_RDS-Logon.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/TerminalServices-RemoteConnectionManager_Op/RemoteConnManger_261_Info_RDS-Conn_Noisy.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/UnkwnChannEID_Med_PossibleHiddenShellcode.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/WLAN-AutoConfig/Sec_8001_Info_Wifi-AP-Conn.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/builtin/WinRM_Op/WinRM_6_Info_SessCreated.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/hayabusa/sysmon/Sysmon_1_Info_ProcExec.yml", + "Error": "yaml: unmarshal errors:\n line 32: mapping key \"sample-evtx\" already defined at line 29" + }, + { + "Path": "hayabusa/hayabusa/sysmon/Sysmon_1_Low_ExeFileRenamed.yml", + "Error": "Invalid modifier endswithfield" + }, + { + "Path": "hayabusa/hayabusa/sysmon/Sysmon_1_Low_ExeFileRenamed.yml", + "Error": "Invalid modifier endswithfield" + }, + { + "Path": "hayabusa/sigma/builtin/emerging-threats/2023/Exploits/CVE-2023-23397/win_smbclient_connectivity_exploit_cve_2023_23397_outlook_remote_file.yml", + "Error": "Missing field mapping 'ServerAddress' in */windows/smbclient-connectivity" + }, + { + "Path": "hayabusa/sigma/builtin/emerging-threats/2023/Malware/Pikabot/proc_creation_win_malware_pikabot_discovery.yml", + "Error": "Missing field mapping 'GrandParentImage' in */windows/security" + }, + { + "Path": "hayabusa/sigma/builtin/security/win_security_rdp_reverse_tunnel.yml", + "Error": "Missing field mapping 'FilterOrigin' in */windows/security" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/proc_creation_win_correlation_multiple_susp_cli.yml", + "Error": "In rule Quick Execution of a Series of Suspicious Commands: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/proc_creation_win_correlation_susp_builtin_commands_recon.yml", + "Error": "In rule Reconnaissance Activity Using BuiltIn Commands: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/sysmon_always_install_elevated_msi_spawned_cmd_and_powershell_spawned_processes.yml", + "Error": "Missing field mapping 'ParentOfParentImage' in */windows/security" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_kernel_and_3rd_party_drivers_exploits_token_stealing.yml", + "Error": "Missing field mapping 'ParentIntegrityLevel' in */windows/security" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_remote_schtask.yml", + "Error": "In rule Remote Schtasks Creation: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_global_catalog_enumeration.yml", + "Error": "In rule Enumeration via the Global Catalog: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_rare_schtasks_creations.yml", + "Error": "In rule Rare Schtasks Creations: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_explicit_credentials.yml", + "Error": "In rule Password Spraying via Explicit Credentials: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_process.yml", + "Error": "In rule Multiple Users Failing to Authenticate from Single Process: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source2.yml", + "Error": "In rule Failed NTLM Logins with Different Accounts from Single Source System: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source_kerberos.yml", + "Error": "In rule Valid Users Failing to Authenticate From Single Source Using Kerberos: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source_kerberos2.yml", + "Error": "In rule Disabled Users Failing To Authenticate From Source Using Kerberos: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source_kerberos3.yml", + "Error": "In rule Invalid Users Failing To Authenticate From Source Using Kerberos: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source_ntlm.yml", + "Error": "In rule Valid Users Failing to Authenticate from Single Source Using NTLM: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_logons_single_source_ntlm2.yml", + "Error": "In rule Invalid Users Failing To Authenticate From Single Source Using NTLM: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_failed_remote_logons_single_source.yml", + "Error": "In rule Multiple Users Remotely Failing To Authenticate From Single Source: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_security_susp_multiple_files_renamed_or_deleted.yml", + "Error": "In rule Suspicious Multiple File Rename Or Delete Occurred: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_susp_failed_hidden_share_mount.yml", + "Error": "In rule Failed Mounting of Hidden Share: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_system_rare_service_installs.yml", + "Error": "In rule Rare Service Installations: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/unsupported/win_taskscheduler_rare_schtask_creation.yml", + "Error": "In rule Rare Scheduled Task Creations: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/builtin/win_alert_mimikatz_keywords.yml", + "Error": "Missing Source: '*/windows/*'" + }, + { + "Path": "hayabusa/sigma/sysmon/create_remote_thread/create_remote_thread_win_susp_uncommon_source_image.yml", + "Error": "Missing field mapping 'TargetParentProcessId' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/emerging-threats/2023/Malware/Pikabot/proc_creation_win_malware_pikabot_discovery.yml", + "Error": "Missing field mapping 'GrandParentImage' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/threat-hunting/create_remote_thread/create_remote_thread_win_powershell_generic.yml", + "Error": "Missing field mapping 'SourceParentImage' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/threat-hunting/process_access/proc_access_win_lsass_uncommon_access_flag.yml", + "Error": "Missing field mapping 'SourceCommandLine' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/dns_query_win_possible_dns_rebinding.yml", + "Error": "In rule Possible DNS Rebinding: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/file_event_executable_and_script_creation_by_office_using_file_ext.yml", + "Error": "Missing field mapping 'FileMagicBytes' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/proc_creation_win_correlation_multiple_susp_cli.yml", + "Error": "In rule Quick Execution of a Series of Suspicious Commands: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/proc_creation_win_correlation_susp_builtin_commands_recon.yml", + "Error": "In rule Reconnaissance Activity Using BuiltIn Commands: Timeframe detections not supported" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/sysmon_always_install_elevated_msi_spawned_cmd_and_powershell_spawned_processes.yml", + "Error": "Missing field mapping 'ParentOfParentImage' in */windows/sysmon" + }, + { + "Path": "hayabusa/sigma/sysmon/unsupported/win_kernel_and_3rd_party_drivers_exploits_token_stealing.yml", + "Error": "Missing field mapping 'ParentIntegrityLevel' in */windows/sysmon" + } + ] +} \ No newline at end of file diff --git a/rules/vql/DOTNET_STARTUP_HOOKS.yml b/rules/vql/DOTNET_STARTUP_HOOKS.yml new file mode 100644 index 0000000..6153243 --- /dev/null +++ b/rules/vql/DOTNET_STARTUP_HOOKS.yml @@ -0,0 +1,32 @@ +title: DotNet Startup Hooks +status: test +description: | + The .NET DLLs listed in the DOTNET_STARTUP_HOOKS environment + variable are loaded into .NET processes at runtime. + +author: Chris Jones - CPIRT, FabFaeb, Antonio Blescia (TheThMando), bmcder02 +references: + - https://persistence-info.github.io/Data/dotnetstartuphooks.html + - https://github.com/last-byte/PersistenceSniper/blob/main/PersistenceSniper/PersistenceSniper.psm1 + +logsource: + product: windows + category: vql + +detection: + selection: + EventData|vql: + x=>x.DOTNET_STARTUP_HOOKS + + condition: selection + +vql: | + x=>dict(EventData=dict(DOTNET_STARTUP_HOOKS={ + SELECT OSPath, Data.value AS Value + FROM glob(globs=[UserEnv, SystemEnv], accessor="registry") + WHERE Value + })) + +vql_args: + UserEnv: HKEY_USERS\*\Environment\DOTNET_STARTUP_HOOKS + SystemEnv: HKEY_LOCAL_MACHINE\System\ControlSet*\Control\Session Manager\Environment\DOTNET_STARTUP_HOOKS diff --git a/rules/vql/DSRMBackdoor.yml b/rules/vql/DSRMBackdoor.yml new file mode 100644 index 0000000..26d8011 --- /dev/null +++ b/rules/vql/DSRMBackdoor.yml @@ -0,0 +1,29 @@ +title: Directory Services Restore Mode Accessible by Local Administrator +status: test +description: | + The password used to enter Directory Services Restore Mode (DSRM) + is the password set to the local administrator of a Domain + Controller during DCPROMO. If the DsrmAdminLogonBehavior property + of the HKLM:\System\CurrentControlSet\Control\Lsa key is set to 2, + this password can be used to access the Domain Controller with the + local administrator account. + +author: Chris Jones - CPIRT, FabFaeb, Antonio Blescia (TheThMando), bmcder02 +references: +- https://github.com/last-byte/PersistenceSniper/blob/main/PersistenceSniper/PersistenceSniper.psm1 +- https://adsecurity.org/?p=1785 + +logsource: + product: windows + category: vql + +detection: + selection: + EventData|vql: + x=>x.DsrmAdminLogonBehavior=2 + + condition: selection + +vql: x=>dict(EventData=dict(DsrmAdminLogonBehavior=GetValue(OSPath=LSAKey))) +vql_args: + LSAKey: HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Lsa\DsrmAdminLogonBehavior diff --git a/rules/vql/OfficeAI.yaml b/rules/vql/OfficeAI.yaml new file mode 100644 index 0000000..428c9f0 --- /dev/null +++ b/rules/vql/OfficeAI.yaml @@ -0,0 +1,36 @@ +title: hijacking of the Microsoft Office AI.exe executable +status: test +description: + Office executables like WINWORD.exe look for AI.exe under the + ProgramFiles/Microsoft Office/root/ and +  ProgramFiles(x86)/Microsoft Office/root/ directories. +  An attacker may place a malicious AI.exe there in order to have persistence +  whenever a user interacts with the Microsoft Office Suite. + +references: +- https://twitter.com/laughing_mantis/status/1645268114966470662 +- https://github.com/last-byte/PersistenceSniper/blob/main/PersistenceSniper/PersistenceSniper.psm1 + +logsource: + category: vql + product: windows +detection: + selection: + EventData|vql: + x=>x.FileLocations + + condition: selection + +level: high +ruletype: VQL + +vql: | + x=>dict(EventData=dict(FileLocations={ + SELECT OSPath, Size, Mtime, + hash(path=OSPath) AS Hash, + authenticode(filename=OSPath) AS Authenticode + FROM glob(globs=Globs) + })) + +vql_args: + Globs: "C:\\Program File*\\Microsoft Office\\root\\Office*\\ai.exe" diff --git a/rules/vql/rclone.yml b/rules/vql/rclone.yml new file mode 100644 index 0000000..56aa1cf --- /dev/null +++ b/rules/vql/rclone.yml @@ -0,0 +1,31 @@ +title: Rclone +status: test +description: Check if RClone was used on this system +logsource: + category: vql + product: windows + +detection: + selection: + "EventData|vql": + x=>x.Files OR x.Registry + + condition: selection + +level: high +ruletype: VQL + +vql: | + x=>dict(EventData=dict( + Files={ + SELECT OSPath, Size, read_file(filename=OSPath, length=100) AS Data + FROM glob(globs=Path, accessor="auto") + }, + Registry=to_dict(item={ + SELECT Name AS _key, Data.value AS _value + FROM glob(globs=Key, accessor="registry") + }))) + +vql_args: + Path: C:\Users\*\AppData\Roaming\rclone\rclone.conf + Key: HKEY_USERS\*\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AppCompatFlags\Compatibility Assistant\Store\*rclone* diff --git a/src/artifact.go b/src/artifact.go index dee2c1d..162b49a 100644 --- a/src/artifact.go +++ b/src/artifact.go @@ -92,5 +92,11 @@ func (self *CompilerContext) WriteArtifact(zip *zip.Writer) error { } fd.Write(MustMarshal(self.config_obj.DefaultDetails.Lookup)) + fd, err = zip.Create("rejected.json") + if err != nil { + return err + } + fd.Write(MustMarshalIndent(self.GetRejected())) + return nil } diff --git a/src/compile.go b/src/compile.go index d74dbb8..9f91c20 100644 --- a/src/compile.go +++ b/src/compile.go @@ -20,15 +20,19 @@ var ( app = kingpin.New("velosigma", "A tool for manipulating sigma files.") - compile_cmd = app.Command("compile", "Compile all the rules into one rule.") - output = compile_cmd.Flag("output", "File to write the artifact bundle to").String() - yaml_output = compile_cmd.Flag("yaml", "File to write the artifact yaml to").String() - config = compile_cmd.Flag("config", "Config file to use").Required().ExistingFile() - level_regex_str = compile_cmd.Flag("level_regex", "A regex to select rule Levels").Default(".").String() + compile_cmd = app.Command("compile", "Compile all the rules into one rule.") + output = compile_cmd.Flag("output", "File to write the artifact bundle to").String() + yaml_output = compile_cmd.Flag("yaml", "File to write the artifact yaml to").String() + rejects_output = compile_cmd.Flag("rejects", "File to write the rejected rules to").String() + ignore_previous_rejects = compile_cmd.Flag("ignore_previous_rejects", "Read the rejects file and ignore any previously known rejected rules").Bool() + config = compile_cmd.Flag("config", "Config file to use").Required().ExistingFile() + level_regex_str = compile_cmd.Flag("level_regex", "A regex to select rule Levels").Default(".").String() debug = app.Flag("debug", "Print more details").Bool() command_handlers []CommandHandler + + allowed_additional_fields = []string{"details", "vql", "vql_args"} ) type CommandHandler func(command string) bool @@ -76,17 +80,27 @@ func (self *CompilerContext) CompileDirs() error { func (self *CompilerContext) CompileRule(rule_yaml, path string) error { rule, err := sigma.ParseRule([]byte(rule_yaml)) if err != nil { + self.addError(err.Error(), path) + if !self.shouldSuppressError(err.Error(), path) { + fmt.Printf("While compiling rule %v: %v\n", path, err) + } return nil } + if rule.Level == "" { + rule.Level = "default" + } + if !self.level_regex.MatchString(rule.Level) { return nil } additional_fields := make(map[string]interface{}) - details := rule.AdditionalFields["details"] - if details != nil { - additional_fields["details"] = details + for _, f := range allowed_additional_fields { + v := rule.AdditionalFields[f] + if v != nil { + additional_fields[f] = v + } } new_rule := sigma.Rule{ @@ -146,7 +160,7 @@ func (self *CompilerContext) CompileRule(rule_yaml, path string) error { return nil } -func doCompile() error { +func doCompile() (err error) { if *yaml_output == "" && *output == "" { return errors.New("Must provide either --output or --yaml") } @@ -164,10 +178,24 @@ func doCompile() error { return fmt.Errorf("Reading Config: %w", err) } + if *ignore_previous_rejects && *rejects_output != "" { + err := context.LoadRejectSupporessions(*rejects_output) + if err != nil { + return fmt.Errorf("Reading Rejects: %w", err) + } + } + defer func() { fmt.Printf("\nGenerated rules with level %v into %v\n", *level_regex_str, *output) - context.Stats() + stats := context.Stats() + + fmt.Printf("\nTotal rules added: %v from %v visited files and %v rejected rules\n", + stats.TotalRules, stats.TotalVisitedRules, stats.TotalErrors) + + if stats.TotalUnhandledErrors > 0 { + err = fmt.Errorf("Unhandled errors %v", stats.TotalUnhandledErrors) + } }() err = context.CompileDirs() @@ -208,6 +236,18 @@ func doCompile() error { out_fd.Write([]byte(artifact)) } + + if *rejects_output != "" { + out_fd, err := os.OpenFile(*rejects_output, + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("Creating yaml output: %w", err) + } + defer out_fd.Close() + + out_fd.Write(MustMarshalIndent(context.GetRejected())) + } + return nil } diff --git a/src/logsources.go b/src/logsources.go index 496bcd2..1ffda56 100644 --- a/src/logsources.go +++ b/src/logsources.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "os" @@ -13,6 +14,13 @@ import ( "github.com/bradleyjkemp/sigma-go" ) +type Stats struct { + TotalUnhandledErrors int + TotalErrors int + TotalRules int + TotalVisitedRules int +} + type CompilerContext struct { logsources map[string]int @@ -29,7 +37,12 @@ type CompilerContext struct { // Map between error reason and the files that were rejected for // it. errored_rules map[string][]string - config_obj *Config + + // We load the previous run's rejected rules to only show + // incremental failures + ignored_rules map[string]bool + + config_obj *Config rules bytes.Buffer level_regex *regexp.Regexp @@ -48,6 +61,7 @@ func NewCompilerContext() *CompilerContext { logsources: make(map[string]int), missing_fields_in_logsources: make(map[string]map[string][]string), errored_rules: make(map[string][]string), + ignored_rules: make(map[string]bool), level_regex: regexp.MustCompile(".*"), fields: make(map[string]int), @@ -93,6 +107,31 @@ func (self *CompilerContext) LoadConfigFromString(data string) error { return nil } +func (self *CompilerContext) LoadRejectSupporessions(filename string) error { + fd, err := os.Open(filename) + if err != nil { + return err + } + defer fd.Close() + + data, err := ioutil.ReadAll(fd) + if err != nil { + return err + } + + rejects := &Rejected{} + err = json.Unmarshal(data, rejects) + if err != nil { + return err + } + + for _, item := range rejects.Rejects { + self.ignored_rules[item.String()] = true + } + + return nil +} + func (self *CompilerContext) Resolve(source_spec string) bool { _, pres := self.config_obj.Sources[source_spec] return pres @@ -105,7 +144,7 @@ func (self *CompilerContext) incLogSource(source_spec string) { self.logsources[source_spec] = count } -func (self *CompilerContext) shouldSuppressError(err_msg string) bool { +func (self *CompilerContext) shouldSuppressError(err_msg string, path string) bool { for _, m := range self.config_obj.BadFieldMappings { if strings.Contains(err_msg, m) { return true @@ -117,11 +156,21 @@ func (self *CompilerContext) shouldSuppressError(err_msg string) bool { return true } } + + key := path + err_msg + _, pres := self.ignored_rules[key] + if pres { + return true + } + return false } -func (self *CompilerContext) Stats() { - total_rules := 0 +func (self *CompilerContext) Stats() Stats { + result := Stats{ + TotalVisitedRules: self.total_visited_rules, + } + sources := []string{} for k := range self.logsources { sources = append(sources, k) @@ -133,20 +182,28 @@ func (self *CompilerContext) Stats() { count, _ := self.logsources[v] fmt.Printf(" %v (%v rules)\n", v, count) - total_rules += count + result.TotalRules += count } - total_reject_rules := 0 if len(self.errored_rules) > 0 { fmt.Printf("\nErrored Rules which were rejected:\n") for k, v := range self.errored_rules { - if self.shouldSuppressError(k) { - continue + messages := []string{} + for _, path := range v { + if self.shouldSuppressError(k, path) { + result.TotalErrors += len(v) + continue + } + + messages = append(messages, fmt.Sprintf(" %v", path)) + result.TotalUnhandledErrors++ } - fmt.Printf(" %v:\n", k) - for _, i := range v { - fmt.Printf(" %v\n", i) - total_reject_rules++ + + if len(messages) > 0 { + fmt.Printf(" %v:\n", k) + for _, m := range messages { + fmt.Println(m) + } } } } @@ -175,10 +232,7 @@ func (self *CompilerContext) Stats() { } - fmt.Printf("\nTotal rules added: %v from %v visited files and %v rejected rules\n", - total_rules, self.total_visited_rules, total_reject_rules) - - return + return result if len(self.missing_fields) > 0 { fmt.Printf("\nMissing field mappings:\n") @@ -199,6 +253,8 @@ func (self *CompilerContext) Stats() { } } } + + return result } func (self *CompilerContext) getSourceFromChannel( @@ -242,6 +298,12 @@ func (self *CompilerContext) check_condition(rule *sigma.Rule) error { }, }) } + + if rule.Detection.Timeframe != "" { + return fmt.Errorf("In rule %v: Timeframe detections not supported", + rule.Title) + } + return nil } diff --git a/src/rejects.go b/src/rejects.go new file mode 100644 index 0000000..eff0f18 --- /dev/null +++ b/src/rejects.go @@ -0,0 +1,35 @@ +package main + +import "sort" + +type RejectedRule struct { + Path string `json:"Path,omitempty"` + Error string `json:"Error,omitempty"` +} + +type Rejected struct { + Rejects []RejectedRule `json:"Rejects"` +} + +func (self RejectedRule) String() string { + return self.Path + self.Error +} + +func (self *CompilerContext) GetRejected() *Rejected { + result := &Rejected{} + + for error_message, paths := range self.errored_rules { + for _, p := range paths { + result.Rejects = append(result.Rejects, RejectedRule{ + Error: error_message, + Path: p, + }) + } + } + + sort.Slice(result.Rejects, func(i, j int) bool { + return result.Rejects[i].Path < result.Rejects[j].Path + }) + + return result +} diff --git a/src/utils.go b/src/utils.go index 4f850c2..d504cf5 100644 --- a/src/utils.go +++ b/src/utils.go @@ -20,6 +20,11 @@ func MustMarshal(in interface{}) []byte { return data } +func MustMarshalIndent(in interface{}) []byte { + data, _ := json.MarshalIndent(in, "", " ") + return data +} + func Dump(in interface{}) { data, _ := json.MarshalIndent(in, "", " ") fmt.Printf("%v", string(data)) From 7588febd0f6fa0209b363344e7dee35c3f46b4e9 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 4 May 2024 16:24:59 +1000 Subject: [PATCH 2/5] Fix tests --- Makefile | 4 +- config/velociraptor_windows_rules.yaml | 11 ++ config/windows_hayabusa_event_monitoring.yaml | 1 + rejected/ChopChopGo_rules_rejects.json | 132 ++++++++++++++++++ .../execution_win_remote_access_tools.yaml | 2 +- .../service_win_remote_access_tools.yaml | 2 +- src/compile.go | 2 +- src/logsources.go | 4 +- src/utils.go | 16 +++ 9 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 rejected/ChopChopGo_rules_rejects.json diff --git a/Makefile b/Makefile index 80e5cf3..5fa33bc 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,10 @@ debugHayabusa: dlv debug ./src -- compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml compileHayabusaMonitoring: - ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --yaml ./output/Velociraptor-Hayabusa-Monitoring.yaml + ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --yaml ./output/Velociraptor-Hayabusa-Monitoring.yaml --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects compileChopChopGo: - ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --yaml ./output/Velociraptor-ChopChopGo-Rules.yaml + ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --yaml ./output/Velociraptor-ChopChopGo-Rules.yaml --rejects rejected/ChopChopGo_rules_rejects.json --ignore_previous_rejects compileCurated: compilePostProcess compileWindowsRules diff --git a/config/velociraptor_windows_rules.yaml b/config/velociraptor_windows_rules.yaml index 917de0b..0848363 100644 --- a/config/velociraptor_windows_rules.yaml +++ b/config/velociraptor_windows_rules.yaml @@ -480,6 +480,17 @@ Sources: channel: - DNS Server + process_creation/windows/*: + query: | + SELECT * FROM parse_evtx(filename=[ + ROOT + "/Microsoft-Windows-Sysmon%4Operational.evtx", + ROOT + "/Security.evtx" + ]) + WHERE System.EventID.Value = 1 OR System.EventID.Value = 4688 + fields: + - Channel + - EventID + process_creation/windows/evtx-execution: query: | SELECT * FROM parse_evtx(filename=[ diff --git a/config/windows_hayabusa_event_monitoring.yaml b/config/windows_hayabusa_event_monitoring.yaml index d99a7aa..6452bd5 100644 --- a/config/windows_hayabusa_event_monitoring.yaml +++ b/config/windows_hayabusa_event_monitoring.yaml @@ -184,6 +184,7 @@ FieldMappings: LogonID: "x=>x.EventData.LogonID" LogonProcessName: "x=>x.EventData.LogonProcessName" LogonType: "x=>x.EventData.LogonType" + Logon_Type: "x=>x.EventData.LogonType" Logon_Account: "x=>x.EventData.Logon_Account" MachineName: "x=>x.EventData.MachineName" MemberName: "x=>x.EventData.MemberName" diff --git a/rejected/ChopChopGo_rules_rejects.json b/rejected/ChopChopGo_rules_rejects.json new file mode 100644 index 0000000..f519d1c --- /dev/null +++ b/rejected/ChopChopGo_rules_rejects.json @@ -0,0 +1,132 @@ +{ + "Rejects": [ + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_binary_padding.yml", + "Error": "Missing field mapping '' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_disable_system_firewall.yml", + "Error": "Missing field mapping 'unit' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_find_cred_in_files.yml", + "Error": "Missing field mapping '' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_network_service_scanning.yml", + "Error": "Missing field mapping 'key' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_omigod_scx_runasprovider_executeshellcommand.yml", + "Error": "Missing field mapping 'uid' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_susp_c2_commands.yml", + "Error": "Missing field mapping 'key' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_web_rce.yml", + "Error": "Missing field mapping 'key' in */linux/auditd" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/auth/lnx_auth_pwnkit_local_privilege_escalation.yml", + "Error": "Missing field mapping '' in */linux/auth" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/clamav/lnx_clamav_relevant_message.yml", + "Error": "Missing Source: '*/linux/clamav'" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/guacamole/lnx_guacamole_susp_guacamole.yml", + "Error": "Missing Source: '*/linux/guacamole'" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/lnx_nimbuspwn_privilege_escalation_exploit.yml", + "Error": "Missing field mapping '' in */linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/sudo/lnx_sudo_cve_2019_14287_user.yml", + "Error": "Missing field mapping 'USER' in */linux/sudo" + }, + { + "Path": "ChopChopGo/rules/linux/builtin/vsftpd/lnx_vsftpd_susp_error_messages.yml", + "Error": "Missing Source: '*/linux/vsftpd'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_doas_conf_creation.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_persistence_cron_files.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_persistence_sudoers_files.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_susp_shell_script_under_profile_directory.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_triple_cross_rootkit_lock_file.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_triple_cross_rootkit_persistence.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_wget_download_file_in_tmp_dir.yml", + "Error": "Missing Source: 'file_event/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_back_connect_shell_dev.yml", + "Error": "Missing Source: 'network_connection/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_crypto_mining_indicators.yml", + "Error": "Missing Source: 'network_connection/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_ngrok_tunnel.yml", + "Error": "Missing Source: 'network_connection/linux/*'" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_cve_2022_26134_atlassian_confluence.yml", + "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_cve_2022_33891_spark_shell_command_injection.yml", + "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_omigod_scx_runasprovider_executescript.yml", + "Error": "Missing field mapping 'User' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_omigod_scx_runasprovider_executeshellcommand.yml", + "Error": "Missing field mapping 'User' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_interactive_bash.yml", + "Error": "Missing field mapping 'ParentCommandLine' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_java_children.yml", + "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_shell_child_process_from_parent_tmp_folder.yml", + "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_system_network_connections_discovery.yml", + "Error": "Missing field mapping 'ParentCommandLine' in process_creation/linux/*" + }, + { + "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_webshell_detection.yml", + "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*" + } + ] +} \ No newline at end of file diff --git a/rules/windows/remote_administration_tools/services/execution_win_remote_access_tools.yaml b/rules/windows/remote_administration_tools/services/execution_win_remote_access_tools.yaml index 85332fa..bc823c6 100644 --- a/rules/windows/remote_administration_tools/services/execution_win_remote_access_tools.yaml +++ b/rules/windows/remote_administration_tools/services/execution_win_remote_access_tools.yaml @@ -21,7 +21,7 @@ logsource: product: windows service: execution detection: - execution_Name|contains: + execution_Name|contains: - 'TeamViewer' - 'GoToAssist' - 'LogMeIn' diff --git a/rules/windows/remote_administration_tools/services/service_win_remote_access_tools.yaml b/rules/windows/remote_administration_tools/services/service_win_remote_access_tools.yaml index aa615cb..f7ed7e8 100644 --- a/rules/windows/remote_administration_tools/services/service_win_remote_access_tools.yaml +++ b/rules/windows/remote_administration_tools/services/service_win_remote_access_tools.yaml @@ -60,7 +60,7 @@ detection: - 'ScreenConnect' - 'SplashTop' - 'OpenSSH' - condition: any + condition: any falsepositives: - Legitimate usage of the tool level: informational diff --git a/src/compile.go b/src/compile.go index 9f91c20..9edbfff 100644 --- a/src/compile.go +++ b/src/compile.go @@ -180,7 +180,7 @@ func doCompile() (err error) { if *ignore_previous_rejects && *rejects_output != "" { err := context.LoadRejectSupporessions(*rejects_output) - if err != nil { + if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Reading Rejects: %w", err) } } diff --git a/src/logsources.go b/src/logsources.go index 1ffda56..5b29c30 100644 --- a/src/logsources.go +++ b/src/logsources.go @@ -392,7 +392,9 @@ func (self *CompilerContext) check_modifiers( func (self *CompilerContext) walk_fields( rule *sigma.Rule, path string, logsource string) (err error) { - for _, search := range rule.Detection.Searches { + + // This needs to be stable so the errors are consistent + for _, search := range ValuesInOrder(rule.Detection.Searches) { for _, event_matcher := range search.EventMatchers { for _, matcher := range event_matcher { // Check if modifiers are valid. diff --git a/src/utils.go b/src/utils.go index d504cf5..e19bfbb 100644 --- a/src/utils.go +++ b/src/utils.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "sort" "strings" ) @@ -29,3 +30,18 @@ func Dump(in interface{}) { data, _ := json.MarshalIndent(in, "", " ") fmt.Printf("%v", string(data)) } + +func ValuesInOrder[V interface{}](in map[string]V) (res []V) { + keys := []string{} + for k := range in { + keys = append(keys, k) + } + + sort.Strings(keys) + for _, k := range keys { + v, _ := in[k] + res = append(res, v) + } + + return res +} From 2be7587e75184a0f03b5d0031e9e9b29b6bbd4f8 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 4 May 2024 16:34:47 +1000 Subject: [PATCH 3/5] Fix test --- .github/workflows/test.yml | 2 ++ src/fixtures/TestCompilation.golden | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f613e8..fe10b45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,9 +37,11 @@ jobs: make linux make compile + # For now we get the latest build from GCP until the 0.72.1 release. - name: Run Velociraptor run: | mv ${{ fromJson(steps.velociraptor.outputs.downloaded_files)[0]}} ./tests/velociraptor + curl -o ./tests/velociraptor https://storage.googleapis.com/releases.velocidex.com/velociraptor/velociraptor-v0.72.1-linux-amd64-musl chmod +x ./tests/velociraptor make golden diff --git a/src/fixtures/TestCompilation.golden b/src/fixtures/TestCompilation.golden index 4ff57da..1e97608 100644 --- a/src/fixtures/TestCompilation.golden +++ b/src/fixtures/TestCompilation.golden @@ -6,6 +6,7 @@ detection: condition: channel_check channel_check: Channel: SomeChannel +level: default --- title: No condition present in rule @@ -18,5 +19,6 @@ detection: Channel: SomeChannel selection: SomeField: XXXX +level: default --- From f2cd665fff636df48edd46bfe1c4871b4debde52 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 4 May 2024 16:37:36 +1000 Subject: [PATCH 4/5] Fix build --- .github/workflows/gh-pages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index f5a981c..7982bc9 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -30,9 +30,9 @@ jobs: - name: Prepare run: | make linux - ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip ; cd . - ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip ; cd . - ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip ; cd . + ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --rejects ./rejected/windows_hayabusa_rejects.json --ignore_previous_rejects + ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects + ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --rejects rejected/ChopChopGo_rules_rejects.json --ignore_previous_rejects cp output/*.zip docs/static/ - name: Build From 116afb26508b314e9b4a0d5bf981e481bf189bb9 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 4 May 2024 16:56:03 +1000 Subject: [PATCH 5/5] Fixed test --- go.mod | 2 ++ go.sum | 5 +++++ src/compile_test.go | 10 ++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b2b599e..9a6aeb6 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,8 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sebdah/goldie/v2 v2.5.3 // indirect + github.com/sergi/go-diff v1.0.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index e1ff0f0..88318c4 100644 --- a/go.sum +++ b/go.sum @@ -19,10 +19,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/src/compile_test.go b/src/compile_test.go index a0bf49a..2b029a8 100644 --- a/src/compile_test.go +++ b/src/compile_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/sebdah/goldie" + "github.com/sebdah/goldie/v2" "github.com/stretchr/testify/assert" ) @@ -90,5 +90,11 @@ func TestCompilation(t *testing.T) { golden += string(context.rules.Bytes()) } - goldie.Assert(t, "TestCompilation", []byte(golden)) + g := goldie.New( + t, + goldie.WithFixtureDir("fixtures"), + goldie.WithDiffEngine(goldie.ClassicDiff), + ) + + g.Assert(t, "TestCompilation", []byte(golden)) }