Keep better track of rejected rules.

.github/workflows/gh-pages.yml

@@ -30,9 +30,9 @@ jobs:
       - name: Prepare
         run: |
           make linux
-          ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip ; cd .
-          ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip ; cd .
-          ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip ; cd .
+          ./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --rejects ./rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
+          ./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
+          ./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --rejects rejected/ChopChopGo_rules_rejects.json --ignore_previous_rejects
           cp output/*.zip docs/static/
 
       - name: Build

.github/workflows/test.yml

@@ -37,9 +37,11 @@ jobs:
         make linux
         make compile
 
+    # For now we get the latest build from GCP until the 0.72.1 release.
     - name: Run Velociraptor
       run: |
         mv ${{ fromJson(steps.velociraptor.outputs.downloaded_files)[0]}} ./tests/velociraptor
+        curl -o ./tests/velociraptor https://storage.googleapis.com/releases.velocidex.com/velociraptor/velociraptor-v0.72.1-linux-amd64-musl
         chmod +x ./tests/velociraptor
         make golden
 

Makefile

@@ -13,16 +13,16 @@ compile: compileThirdParty compileCurated
 compileThirdParty:  compileHayabusa compileHayabusaMonitoring compileChopChopGo
 
 compileHayabusa:
-	./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml
+	./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
 
 debugHayabusa:
 	dlv debug ./src -- compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --yaml ./output/Velociraptor-Hayabusa-Rules.yaml
 
 compileHayabusaMonitoring:
-	./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --yaml ./output/Velociraptor-Hayabusa-Monitoring.yaml
+	./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --yaml ./output/Velociraptor-Hayabusa-Monitoring.yaml --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
 
 compileChopChopGo:
-	./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --yaml ./output/Velociraptor-ChopChopGo-Rules.yaml
+	./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --yaml ./output/Velociraptor-ChopChopGo-Rules.yaml --rejects rejected/ChopChopGo_rules_rejects.json --ignore_previous_rejects
 
 compileCurated:  compilePostProcess compileWindowsRules
 

config/velociraptor_windows_rules.yaml

@@ -480,6 +480,17 @@ Sources:
     channel:
       - DNS Server
 
+  process_creation/windows/*:
+    query: |
+      SELECT * FROM parse_evtx(filename=[
+         ROOT + "/Microsoft-Windows-Sysmon%4Operational.evtx",
+         ROOT + "/Security.evtx"
+      ])
+      WHERE System.EventID.Value = 1 OR System.EventID.Value = 4688
+    fields:
+      - Channel
+      - EventID
+
   process_creation/windows/evtx-execution:
     query: |
       SELECT * FROM parse_evtx(filename=[

config/windows_hayabusa_event_monitoring.yaml

@@ -184,6 +184,7 @@ FieldMappings:
   LogonID: "x=>x.EventData.LogonID"
   LogonProcessName: "x=>x.EventData.LogonProcessName"
   LogonType: "x=>x.EventData.LogonType"
+  Logon_Type: "x=>x.EventData.LogonType"
   Logon_Account: "x=>x.EventData.Logon_Account"
   MachineName: "x=>x.EventData.MachineName"
   MemberName: "x=>x.EventData.MemberName"

config/windows_hayabusa_rules.yaml

@@ -1,7 +1,7 @@
 Preamble: |
   name: Windows.Hayabusa.Rules
   description: |
-    This artifact compiles the Hayabusa rulese into a Velociraptor
+    This artifact compiles the Hayabusa ruleset into a Velociraptor
     artifact using the internal Velociraptor sigma engine.
 
     To read more about this artifact see
@@ -77,6 +77,16 @@ Preamble: |
         `%%1937`= 'ELEVATED_TOKEN',
         `%%1938`= 'LIMITED_TOKEN')
 
+      // Helpers for VQL rules
+      LET FetchKeyValues(OSPath) = to_dict(item={
+        SELECT Name AS _key, Data.value AS _value
+        FROM glob(globs="*", accessor="registry", root=OSPath)
+      })
+
+      LET GetValue(OSPath) = stat(filename=OSPath, accessor="registry").Data.value
+      LET Hostname <= dict(H={ SELECT Hostname FROM info()}).H[0].Hostname
+
+
 FieldMappings:
   MandatoryLabel: |
     x=>get(item=MandatoryLabelLookup, member=x.EventData.MandatoryLabel || "-")
@@ -176,6 +186,7 @@ FieldMappings:
   LogonID: "x=>x.EventData.LogonID"
   LogonProcessName: "x=>x.EventData.LogonProcessName"
   LogonType: "x=>x.EventData.LogonType"
+  Logon_Type: "x=>x.EventData.LogonType"
   Logon_Account: "x=>x.EventData.Logon_Account"
   MachineName: "x=>x.EventData.MachineName"
   MemberName: "x=>x.EventData.MemberName"
@@ -365,6 +376,8 @@ FieldMappings:
   # antivirus/windows/windefend
   Feature_Name: "x=>x.EventData.`Feature Name`"
 
+  EventData: "x=>x.EventData"
+
 DefaultDetails:
   Query: |
     x=>get(item=DefaultDetails,
@@ -921,6 +934,16 @@ Sources:
       - Channel
       - EventID
 
+  vql/windows/*:
+    query: |
+      SELECT dict(
+          Computer=Hostname,
+          Channel="VQL Evaluation",
+          TimeCreated=dict(SystemTime=now())
+        ) AS System,
+        dict() AS EventData
+      FROM scope()
+
 QueryTemplate: |
    LET Rules <= SigmaRules || gunzip(string=base64decode(string="{{.Base64CompressedRules}}"))
    LET FieldMapping <= parse_json(data=gunzip(string=base64decode(string="{{.Base64FieldMapping}}")))
@@ -938,7 +961,7 @@ QueryTemplate: |
           dict(System=System,
                EventData=X.EventData || X.UserData,
                Message=X.Message) AS _Event,
-          _Match
+          _Match, *
     FROM sigma(
       rules=split(string= Rules, sep_string="\n---\n"),
       log_sources= LogSources, debug=Debug,
@@ -977,6 +1000,10 @@ BadFieldMappings:
 # hayabusa/sigma/builtin/security/win_security_rdp_reverse_tunnel.yml
 - FilterOrigin
 
+# We do not currently support rules with Timeframe in them so we
+# silently drop these.
+- Timeframe detections not supported
+
 
 # Many rules are broken and have bad log source section. The following
 # list suppresses these warnings (but the rules are still rejected)
@@ -997,3 +1024,4 @@ RuleDirectories:
   - hayabusa/hayabusa/sysmon/
   - hayabusa/sigma/builtin/
   - hayabusa/sigma/sysmon/
+  - rules/vql/

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.3.2
 	github.com/bradleyjkemp/sigma-go v0.6.4
 	github.com/davecgh/go-spew v1.1.1
+	github.com/sebdah/goldie v1.0.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	gopkg.in/yaml.v3 v3.0.1
@@ -17,7 +18,12 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/sebdah/goldie v1.0.0 // indirect
+	github.com/sebdah/goldie/v2 v2.5.3 // indirect
+	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 )
+
+replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed
+
+//replace github.com/bradleyjkemp/sigma-go => ../sigma-go

go.sum

@@ -1,3 +1,5 @@
+github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed h1:zqhuWeg6oqO3jNabjKJaGO7DreiGhbVfeyqleICMAZk=
+github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4=
 github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No=
 github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4=
 github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU=
@@ -8,8 +10,6 @@ github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1p
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj0JTv4mTs=
-github.com/bradleyjkemp/sigma-go v0.6.4 h1:J6Sqwbgv7wsEuP7xbsG8dvTrTc9lhkf5BvYF+gO9vzc=
-github.com/bradleyjkemp/sigma-go v0.6.4/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -19,10 +19,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
 github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

rejected/ChopChopGo_rules_rejects.json

@@ -0,0 +1,132 @@
+{
+ "Rejects": [
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_binary_padding.yml",
+   "Error": "Missing field mapping '' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_disable_system_firewall.yml",
+   "Error": "Missing field mapping 'unit' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_find_cred_in_files.yml",
+   "Error": "Missing field mapping '' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_network_service_scanning.yml",
+   "Error": "Missing field mapping 'key' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_omigod_scx_runasprovider_executeshellcommand.yml",
+   "Error": "Missing field mapping 'uid' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_susp_c2_commands.yml",
+   "Error": "Missing field mapping 'key' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/auditd/lnx_auditd_web_rce.yml",
+   "Error": "Missing field mapping 'key' in */linux/auditd"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/auth/lnx_auth_pwnkit_local_privilege_escalation.yml",
+   "Error": "Missing field mapping '' in */linux/auth"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/clamav/lnx_clamav_relevant_message.yml",
+   "Error": "Missing Source: '*/linux/clamav'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/guacamole/lnx_guacamole_susp_guacamole.yml",
+   "Error": "Missing Source: '*/linux/guacamole'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/lnx_nimbuspwn_privilege_escalation_exploit.yml",
+   "Error": "Missing field mapping '' in */linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/sudo/lnx_sudo_cve_2019_14287_user.yml",
+   "Error": "Missing field mapping 'USER' in */linux/sudo"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/builtin/vsftpd/lnx_vsftpd_susp_error_messages.yml",
+   "Error": "Missing Source: '*/linux/vsftpd'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_doas_conf_creation.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_persistence_cron_files.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_persistence_sudoers_files.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_susp_shell_script_under_profile_directory.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_triple_cross_rootkit_lock_file.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_triple_cross_rootkit_persistence.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/file_event/file_event_lnx_wget_download_file_in_tmp_dir.yml",
+   "Error": "Missing Source: 'file_event/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_back_connect_shell_dev.yml",
+   "Error": "Missing Source: 'network_connection/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_crypto_mining_indicators.yml",
+   "Error": "Missing Source: 'network_connection/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/network_connection/net_connection_lnx_ngrok_tunnel.yml",
+   "Error": "Missing Source: 'network_connection/linux/*'"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_cve_2022_26134_atlassian_confluence.yml",
+   "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_cve_2022_33891_spark_shell_command_injection.yml",
+   "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_omigod_scx_runasprovider_executescript.yml",
+   "Error": "Missing field mapping 'User' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_omigod_scx_runasprovider_executeshellcommand.yml",
+   "Error": "Missing field mapping 'User' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_interactive_bash.yml",
+   "Error": "Missing field mapping 'ParentCommandLine' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_java_children.yml",
+   "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_susp_shell_child_process_from_parent_tmp_folder.yml",
+   "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_system_network_connections_discovery.yml",
+   "Error": "Missing field mapping 'ParentCommandLine' in process_creation/linux/*"
+  },
+  {
+   "Path": "ChopChopGo/rules/linux/process_creation/proc_creation_lnx_webshell_detection.yml",
+   "Error": "Missing field mapping 'ParentImage' in process_creation/linux/*"
+  }
+ ]
+}