Merge branch 'main' into adelhajhassan/remove_stale_unit_test

.github/workflows/serverless-integration.yml

@@ -30,7 +30,7 @@ jobs:
           path: go/src/github.com/DataDog/datadog-agent
 
       - name: Set up Node 20
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
 

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 08b9d8e048c1
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 7e55b9e3279a

.pre-commit-config.yaml

@@ -116,6 +116,14 @@ repos:
       files: .*gitlab.*\.yml$
       pass_filenames: false
       stages: [pre-push]
+    - id: gitlab-lint-jobs-codeowners
+      name: gitlab-lint-jobs-codeowners
+      description: lint the gitlab configuration to verify jobs codeowners
+      entry: 'inv linter.gitlab-ci-jobs-codeowners'
+      language: system
+      require_serial: true
+      files: .*gitlab.*\.yml$
+      pass_filenames: false
     - id: update-go
       name: update-go
       description: test formatting of files will allow go update

cmd/agent/subcommands/snmp/command.go

@@ -11,14 +11,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/DataDog/datadog-agent/comp/serializer/compression/compressionimpl"
 	"net"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/DataDog/datadog-agent/comp/serializer/compression/compressionimpl"
-
 	"github.com/gosnmp/gosnmp"
 	"github.com/spf13/cobra"
 	"go.uber.org/fx"
@@ -177,8 +176,9 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 	snmpCmd.AddCommand(snmpWalkCmd)
 
 	snmpScanCmd := &cobra.Command{
-		Use:   "scan <ipaddress>[:port]",
-		Short: "Scan a device for the profile editor.",
+		Hidden: true,
+		Use:    "scan <ipaddress>[:port]",
+		Short:  "Scan a device for the profile editor.",
 		Long: `Walk the SNMP tree for a device, collecting available OIDs.
 		Flags that aren't specified will be pulled from the agent SNMP config if possible.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -229,7 +229,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 	snmpScanCmd.Flags().BoolVar(&connParams.UseUnconnectedUDPSocket, "use-unconnected-udp-socket", defaultUseUnconnectedUDPSocket, "If specified, changes net connection to be unconnected UDP socket")
 
 	// This command does nothing until the backend supports it, so it isn't enabled yet.
-	// snmpCmd.AddCommand(snmpScanCmd)
+	snmpCmd.AddCommand(snmpScanCmd)
 
 	return []*cobra.Command{snmpCmd}
 }

cmd/security-agent/subcommands/runtime/command.go

@@ -551,6 +551,16 @@ func eventDataFromJSON(file string) (eval.Event, error) {
 			if err := event.SetFieldValue(k, int(value)); err != nil {
 				return nil, err
 			}
+		case []any:
+			if stringSlice, ok := anySliceToStringSlice(v); ok {
+				if err := event.SetFieldValue(k, stringSlice); err != nil {
+					return nil, err
+				}
+			} else {
+				if err := event.SetFieldValue(k, v); err != nil {
+					return nil, err
+				}
+			}
 		default:
 			if err := event.SetFieldValue(k, v); err != nil {
 				return nil, err
@@ -561,6 +571,18 @@ func eventDataFromJSON(file string) (eval.Event, error) {
 	return event, nil
 }
 
+func anySliceToStringSlice(in []any) ([]string, bool) {
+	out := make([]string, len(in))
+	for i, v := range in {
+		val, ok := v.(string)
+		if !ok {
+			return nil, false
+		}
+		out[i] = val
+	}
+	return out, true
+}
+
 func evalRule(_ log.Component, _ config.Component, _ secrets.Component, evalArgs *evalCliParams) error {
 	policiesDir := evalArgs.dir
 

pkg/security/probe/probe.go

@@ -47,6 +47,7 @@ type PlatformProbe interface {
 	DumpDiscarders() (string, error)
 	FlushDiscarders() error
 	ApplyRuleSet(_ *rules.RuleSet) (*kfilters.ApplyRuleSetReport, error)
+	OnNewRuleSetLoaded(_ *rules.RuleSet)
 	OnNewDiscarder(_ *rules.RuleSet, _ *model.Event, _ eval.Field, _ eval.EventType)
 	HandleActions(_ *eval.Context, _ *rules.Rule)
 	NewEvent() *model.Event
@@ -229,10 +230,15 @@ func (p *Probe) FlushDiscarders() error {
 
 // ApplyRuleSet setup the probes for the provided set of rules and returns the policy report.
 func (p *Probe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetReport, error) {
+	return p.PlatformProbe.ApplyRuleSet(rs)
+}
+
+// OnNewRuleSetLoaded resets statistics and states once a new rule set is loaded
+func (p *Probe) OnNewRuleSetLoaded(rs *rules.RuleSet) {
 	p.ruleActionStatsLock.Lock()
 	clear(p.ruleActionStats)
 	p.ruleActionStatsLock.Unlock()
-	return p.PlatformProbe.ApplyRuleSet(rs)
+	p.PlatformProbe.OnNewRuleSetLoaded(rs)
 }
 
 // Snapshot runs the different snapshot functions of the resolvers that

pkg/security/probe/probe_ebpf.go

@@ -1648,8 +1648,6 @@ func (p *EBPFProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRepor
 	// activity dump & security profiles
 	needRawSyscalls := p.isNeededForActivityDump(model.SyscallsEventType.String())
 
-	p.processKiller.Apply(rs)
-
 	// kill action
 	if p.config.RuntimeSecurity.EnforcementEnabled && isKillActionPresent(rs) {
 		if !p.config.RuntimeSecurity.EnforcementRawSyscallEnabled {
@@ -1687,6 +1685,11 @@ func (p *EBPFProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRepor
 	return ars, nil
 }
 
+// OnNewRuleSetLoaded resets statistics and states once a new rule set is loaded
+func (p *EBPFProbe) OnNewRuleSetLoaded(rs *rules.RuleSet) {
+	p.processKiller.Reset(rs)
+}
+
 // NewEvent returns a new event
 func (p *EBPFProbe) NewEvent() *model.Event {
 	return NewEBPFEvent(p.fieldHandlers)

pkg/security/probe/probe_ebpfless.go

@@ -580,11 +580,15 @@ func (p *EBPFLessProbe) FlushDiscarders() error {
 }
 
 // ApplyRuleSet applies the new ruleset
-func (p *EBPFLessProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetReport, error) {
-	p.processKiller.Apply(rs)
+func (p *EBPFLessProbe) ApplyRuleSet(_ *rules.RuleSet) (*kfilters.ApplyRuleSetReport, error) {
 	return &kfilters.ApplyRuleSetReport{}, nil
 }
 
+// OnNewRuleSetLoaded resets statistics and states once a new rule set is loaded
+func (p *EBPFLessProbe) OnNewRuleSetLoaded(rs *rules.RuleSet) {
+	p.processKiller.Reset(rs)
+}
+
 // HandleActions handles the rule actions
 func (p *EBPFLessProbe) HandleActions(ctx *eval.Context, rule *rules.Rule) {
 	ev := ctx.Event.(*model.Event)

pkg/security/probe/probe_windows.go

@@ -1265,8 +1265,6 @@ func (p *WindowsProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRe
 		}
 	}
 
-	p.processKiller.Apply(rs)
-
 	ars, err := kfilters.NewApplyRuleSetReport(p.config.Probe, rs)
 	if err != nil {
 		return nil, err
@@ -1291,6 +1289,11 @@ func (p *WindowsProbe) ApplyRuleSet(rs *rules.RuleSet) (*kfilters.ApplyRuleSetRe
 	return ars, nil
 }
 
+// OnNewRuleSetLoaded resets statistics and states once a new rule set is loaded
+func (p *WindowsProbe) OnNewRuleSetLoaded(rs *rules.RuleSet) {
+	p.processKiller.Reset(rs)
+}
+
 // FlushDiscarders invalidates all the discarders
 func (p *WindowsProbe) FlushDiscarders() error {
 	p.discardedPaths.Purge()

pkg/security/probe/process_killer.go

@@ -267,8 +267,8 @@ func (p *ProcessKiller) KillAndReport(kill *rules.KillDefinition, rule *rules.Ru
 	return true
 }
 
-// Apply applies to ruleset to the process killer
-func (p *ProcessKiller) Apply(rs *rules.RuleSet) {
+// Reset the state and statistics of the process killer
+func (p *ProcessKiller) Reset(rs *rules.RuleSet) {
 	if p.cfg.RuntimeSecurity.EnforcementEnabled {
 		var ruleSetHasKillAction bool
 		var rulesetHasKillDisarmer bool

pkg/security/rules/engine.go

@@ -333,6 +333,9 @@ func (e *RuleEngine) LoadPolicies(providers []rules.PolicyProvider, sendLoadedRe
 		return fmt.Errorf("failed to flush discarders: %w", err)
 	}
 
+	// reset the probe process killer state once the new ruleset is loaded
+	e.probe.OnNewRuleSetLoaded(rs)
+
 	content, _ := json.Marshal(report)
 	seclog.Debugf("Policy report: %s", content)
 

tasks/libs/common/git.py

@@ -72,7 +72,7 @@ def get_file_modifications(
     flags = '--no-renames' if no_renames else ''
 
     modifications = [
-        line.split()
+        line.split('\t')
         for line in ctx.run(f"git diff --name-status {flags} {last_main_commit}", hide=True).stdout.splitlines()
     ]
 

test/fakeintake/client/client.go

@@ -776,7 +776,7 @@ func (c *Client) get(route string) ([]byte, error) {
 	err := backoff.Retry(func() error {
 		tmpResp, err := http.Get(fmt.Sprintf("%s/%s", c.fakeIntakeURL, route))
 		if err, ok := err.(net.Error); ok && err.Timeout() {
-			panic("fakeintake call timed out")
+			panic(fmt.Sprintf("fakeintake call timed out: %v", err))
 		}
 		if err != nil {
 			return err

test/new-e2e/go.mod

@@ -59,7 +59,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20241007132100-08b9d8e048c1
+	github.com/DataDog/test-infra-definitions v0.0.0-20241010155348-7e55b9e3279a
 	github.com/aws/aws-sdk-go-v2 v1.32.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.40
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2