diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ad6aeb1dde6e..5960e3f557a6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ experimental: templates: job_template: &job_template docker: - - image: gcr.io/datadoghq/agent-circleci-runner:v44534774-f5cc3e24 + - image: gcr.io/datadoghq/agent-circleci-runner:v45979046-89a73a40 environment: USE_SYSTEM_LIBS: "1" working_directory: /go/src/github.com/DataDog/datadog-agent diff --git a/.copyright-overrides.yml b/.copyright-overrides.yml index 1e31d179ece4c..9fbeffdec2351 100644 --- a/.copyright-overrides.yml +++ b/.copyright-overrides.yml @@ -28,7 +28,6 @@ github.com/elastic/*: Copyright 2017-2018 Elasticsearch B.V. github.com/golang/*: Copyright (c) 2009 The Go Authors. All rights reserved. github.com/googleapis/*: Copyright 2017 Google Inc. github.com/google/go-cmp/*: Copyright (c) 2017 The Go Authors. All rights reserved. -github.com/grpc-ecosystem/*: Copyright (c) 2015, Gengo, Inc. github.com/hashicorp/*: Copyright © 2014-2018 HashiCorp, Inc github.com/prometheus/*: Copyright 2012-2015 The Prometheus Authors github.com/open-telemetry/*: Copyright The OpenTelemetry Authors @@ -44,8 +43,9 @@ sigs.k8s.io/*: Copyright 2017 The Kubernetes Authors. contrib.go.opencensus.io/exporter/prometheus: Copyright 2020, OpenCensus Authors go.opencensus.io: Copyright 2018, OpenCensus Authors go.opencensus.io/*: Copyright 2018, OpenCensus Authors -github.com/grpc-ecosystem/*: Copyright 2016 Michal Witkowski. All Rights Reserved. -google.golang.org/genproto/*: Copyright 2015 Google LLC +github.com/grpc-ecosystem/go-grpc-prometheus: Copyright 2016 Michal Witkowski. All Rights Reserved. +github.com/grpc-ecosystem/go-grpc-middleware: Copyright (c) The go-grpc-middleware Authors. +github.com/grpc-ecosystem/grpc-gateway: Copyright (c) 2015, Gengo, Inc. google.golang.org/grpc: Copyright 2014 gRPC authors. google.golang.org/grpc/*: Copyright 2014 gRPC authors. github.com/skydive-project/go-debouncer: Copyright (C) 2018 Red Hat, Inc. @@ -213,6 +213,7 @@ github.com/gogo/protobuf: ["Copyright (c) 2013, The GoGo Authors. All rights res # * Automatic "Copyright (c)" # * Year is based on the year of the first commit to the repository # * Author, when known only by GitHub username, is "GitHub Full Name (github-username)" +github.com/moby/docker-image-spec/specs-go/v1: Copyright 2012-2017 Docker, Inc. github.com/moby/sys/mountinfo: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved. github.com/moby/sys/signal: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved. github.com/moby/sys/user: Copyright (c) 2014-2018 The Docker & Go Authors. All rights reserved. @@ -333,6 +334,7 @@ github.com/pjbgf/sha1cd: - Copyright 2009 The Go Authors. All rights reserved. - Copyright 2017 Marc Stevens , Dan Shumow - Copyright 2022 Paulo Gomes + - Copyright 2023 pjbgf github.com/kr/pretty: Copyright 2012 Keith Rarick github.com/kr/text: Copyright 2012 Keith Rarick @@ -343,7 +345,6 @@ github.com/openvex/go-vex: Copyright 2023 The OpenVEX Authors # Copyright information is in the LICENSE github.com/aquasecurity/trivy-policies/pkg/spec: Copyright (c) 2024 Aqua Security github.com/aquasecurity/trivy-policies/rules/specs: Copyright (c) 2024 Aqua Security -github.com/pjbgf/sha1cd: Copyright 2023 pjbgf github.com/go-errors/errors: Copyright (c) 2015 Conrad Irwin github.com/apache/arrow/*: Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) github.com/tailscale/tailscale: Copyright (c) 2020 Tailscale Inc & AUTHORS. diff --git a/.ddqa/config.toml b/.ddqa/config.toml index e24656da4b464..0b2d11709b4b1 100644 --- a/.ddqa/config.toml +++ b/.ddqa/config.toml @@ -216,3 +216,10 @@ jira_issue_type = "Task" jira_statuses = ["To Do", "In Progress", "Done"] github_team = "agent-release-management" github_labels = ["team/agent-release-management"] + +[teams."Dynamic Instrumentation"] +jira_project = "DEBUG" +jira_issue_type = "Task" +jira_statuses = ["To Do", "In Progress", "Done"] +github_team = "debugger-go" +github_labels = ["team/dynamic-instrumentation"] \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index acee6a8d7bea4..0b7a4b14e7cf1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,6 +8,7 @@ go.sum -diff -merge linguist-generated=true *_easyjson.go -diff -merge *_easyjson.go linguist-generated=true pkg/security/probe/constantfetch/btfhub/constants.json -diff -merge linguist-generated=true +pkg/security/seclwin/** -diff -merge linguist-generated=true # Fixtures should have LF line endings because they are checked against OCI packages built on Linux pkg/fleet/internal/fixtures/** text=auto eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bfbd0ebb033a1..d946708dd239e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -49,10 +49,10 @@ /.github/CODEOWNERS # do not notify anyone /.github/*_TEMPLATE.md @DataDog/agent-devx-loops /.github/dependabot.yaml @DataDog/agent-devx-infra -/.github/workflows/serverless-benchmarks.yml @DataDog/serverless -/.github/workflows/serverless-binary-size.yml @DataDog/serverless -/.github/workflows/serverless-integration.yml @DataDog/serverless -/.github/workflows/serverless-vuln-scan.yml @DataDog/serverless +/.github/workflows/serverless-benchmarks.yml @DataDog/serverless @Datadog/serverless-aws +/.github/workflows/serverless-binary-size.yml @DataDog/serverless @Datadog/serverless-aws +/.github/workflows/serverless-integration.yml @DataDog/serverless @Datadog/serverless-aws +/.github/workflows/serverless-vuln-scan.yml @DataDog/serverless @Datadog/serverless-aws /.github/workflows/windows-*.yml @DataDog/windows-agent /.github/workflows/cws-btfhub-sync.yml @DataDog/agent-security /.github/workflows/gohai.yml @DataDog/agent-shared-components @@ -66,13 +66,54 @@ # Files containing job contents are owned by teams in charge of the jobs + agent-devx-infra or agent-delivery # Files that only describe structure (eg. includes, rules) are owned by agent-devx-infra -/.gitlab/ @DataDog/agent-devx-infra +/.gitlab/.ci-linters.yml @DataDog/agent-devx-infra +/.gitlab/.pre/* @DataDog/agent-devx-infra +/.gitlab/check_deploy/* @DataDog/agent-delivery +/.gitlab/check_merge/* @DataDog/agent-devx-infra +/.gitlab/deploy*/* @DataDog/agent-delivery +/.gitlab/deps_fetch/* @DataDog/agent-devx-infra +/.gitlab/e2e/* @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/e2e_install_packages/* @DataDog/agent-delivery +/.gitlab/e2e_pre_test/* @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/junit_upload/* @DataDog/agent-devx-infra +/.gitlab/kernel_matrix_testing/* @DataDog/agent-devx-infra @DataDog/ebpf-platform +/.gitlab/lint/* @DataDog/agent-devx-infra +/.gitlab/maintenance_jobs/* @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/notify/* @DataDog/agent-devx-infra +/.gitlab/pkg_metrics/* @DataDog/agent-devx-infra +/.gitlab/post_rc_build/* @DataDog/agent-devx-infra +/.gitlab/setup/* @DataDog/agent-devx-infra +/.gitlab/trigger_release/* @DataDog/agent-devx-infra + +/.gitlab/binary_build/cws_instrumentation.yml @DataDog/agent-devx-infra @DataDog/agent-security +/.gitlab/binary_build/include.yml @DataDog/agent-devx-infra +/.gitlab/binary_build/linux.yml @DataDog/agent-devx-infra @DataDog/agent-delivery +/.gitlab/functional_test/include.yml @DataDog/agent-devx-infra +/.gitlab/functional_test_cleanup/functional_test_cleanup.yml @DataDog/agent-devx-infra @DataDog/agent-security +/.gitlab/install_script_testing/install_script_testing.yml @DataDog/agent-delivery +/.gitlab/integration_test/dogstatsd.yml @DataDog/agent-devx-infra @DataDog/agent-metrics-logs +/.gitlab/integration_test/include.yml @DataDog/agent-devx-infra +/.gitlab/integration_test/linux.yml @DataDog/agent-devx-infra +/.gitlab/integration_test/otel.yml @DataDog/agent-devx-infra @DataDog/opentelemetry +/.gitlab/internal_image_deploy/internal_image_deploy.yml @DataDog/agent-delivery +/.gitlab/internal_kubernetes_deploy/include.yml @DataDog/agent-devx-infra +/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml @DataDog/agent-delivery +/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml @DataDog/agent-delivery +/.gitlab/package_deps_build/package_deps_build.yml @DataDog/agent-devx-infra @DataDog/ebpf-platform +/.gitlab/powershell_script_signing/powershell_script_signing.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/source_test/golang_deps_diff.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/source_test/include.yml @DataDog/agent-devx-infra +/.gitlab/source_test/linux.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/source_test/macos.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/source_test/notify.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/source_test/slack.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops +/.gitlab/source_test/tooling_unit_tests.yml @DataDog/agent-devx-infra @DataDog/agent-devx-loops /.gitlab/binary_build/cluster_agent_cloudfoundry.yml @DataDog/platform-integrations @DataDog/agent-delivery /.gitlab/binary_build/cluster_agent.yml @DataDog/container-integrations @DataDog/agent-delivery /.gitlab/binary_build/fakeintake.yml @DataDog/agent-devx-loops /.gitlab/binary_build/otel_agent.yml @DataDog/opentelemetry @DataDog/agent-delivery -/.gitlab/binary_build/serverless.yml @DataDog/serverless @DataDog/agent-delivery +/.gitlab/binary_build/serverless.yml @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-delivery /.gitlab/binary_build/system_probe.yml @DataDog/ebpf-platform @DataDog/agent-delivery /.gitlab/binary_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent @@ -98,18 +139,19 @@ /.gitlab/common/test_infra_version.yml @DataDog/agent-devx-loops @DataDog/agent-devx-infra /.gitlab/e2e/e2e.yml @DataDog/container-integrations @DataDog/agent-devx-loops -/.gitlab/e2e_k8s/e2e_k8s.yml @DataDog/container-integrations @DataDog/agent-devx-loops /.gitlab/e2e/install_packages @DataDog/agent-delivery /.gitlab/container_build/fakeintake.yml @DataDog/agent-e2e-testing @DataDog/agent-devx-loops /.gitlab/binary_build/fakeintake.yml @DataDog/agent-e2e-testing @DataDog/agent-devx-loops /.gitlab/functional_test/security_agent.yml @DataDog/agent-security @DataDog/agent-devx-infra -/.gitlab/functional_test/serverless.yml @DataDog/serverless @DataDog/agent-devx-infra +/.gitlab/functional_test/serverless.yml @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-devx-infra /.gitlab/functional_test_cleanup.yml @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra /.gitlab/functional_test/system_probe_windows.yml @DataDog/agent-devx-infra @DataDog/windows-kernel-integrations /.gitlab/functional_test/common.yml @DataDog/agent-devx-infra @DataDog/windows-kernel-integrations /.gitlab/functional_test/oracle.yml @DataDog/agent-devx-infra @DataDog/database-monitoring +/.gitlab/powershell_script_deploy/powershell_script_deploy.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/powershell_script_deploy/powershell_script_signing.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/choco_build/choco_build.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/choco_deploy/choco_deploy.yml @DataDog/agent-delivery @DataDog/windows-agent @@ -188,7 +230,7 @@ /cmd/dogstatsd/ @DataDog/agent-metrics-logs /cmd/otel-agent/ @DataDog/opentelemetry /cmd/process-agent/ @DataDog/processes -/cmd/serverless/ @DataDog/serverless +/cmd/serverless/ @DataDog/serverless @Datadog/serverless-aws /cmd/serverless/dependencies*.txt @DataDog/serverless @DataDog/agent-shared-components /cmd/serverless-init/ @DataDog/serverless /cmd/system-probe/ @DataDog/ebpf-platform @@ -242,6 +284,7 @@ /Makefile.trace @DataDog/agent-apm /omnibus/ @DataDog/agent-delivery +/omnibus/python-scripts/ @DataDog/platform-integrations /omnibus/config/patches/openscap/ @DataDog/agent-cspm /omnibus/config/software/datadog-agent-integrations-*.rb @DataDog/agent-integrations /omnibus/config/software/datadog-security-agent*.rb @Datadog/agent-security @DataDog/agent-delivery @@ -306,12 +349,13 @@ /pkg/errors/ @DataDog/agent-shared-components /pkg/forwarder/ @DataDog/agent-processing-and-routing /pkg/gohai @DataDog/agent-shared-components +/pkg/gpu/ @DataDog/ebpf-platform /pkg/jmxfetch/ @DataDog/agent-metrics-logs /pkg/metrics/ @DataDog/agent-metrics-logs /pkg/metrics/metricsource.go @DataDog/agent-metrics-logs @DataDog/agent-integrations /pkg/serializer/ @DataDog/agent-processing-and-routing /pkg/serializer/internal/metrics/origin_mapping.go @DataDog/agent-processing-and-routing @DataDog/agent-integrations -/pkg/serverless/ @DataDog/serverless +/pkg/serverless/ @DataDog/serverless @Datadog/serverless-aws /pkg/serverless/appsec/ @DataDog/asm-go /pkg/status/ @DataDog/agent-shared-components /pkg/status/templates/trace-agent.tmpl @DataDog/agent-apm @@ -325,7 +369,7 @@ /pkg/trace/api/otlp*.go @DataDog/opentelemetry /pkg/trace/traceutil/otel*.go @DataDog/opentelemetry /pkg/trace/stats/ @DataDog/agent-apm @DataDog/opentelemetry -/pkg/trace/telemetry/ @DataDog/telemetry-and-analytics +/pkg/trace/telemetry/ @DataDog/apm-trace-storage /comp/core/autodiscovery/listeners/ @DataDog/container-platform /comp/core/autodiscovery/listeners/cloudfoundry*.go @DataDog/platform-integrations /comp/core/autodiscovery/listeners/snmp*.go @DataDog/network-device-monitoring @@ -340,7 +384,7 @@ /pkg/clusteragent/admission/mutate/autoscaling @DataDog/container-integrations /pkg/clusteragent/admission/mutate/autoinstrumentation/ @DataDog/container-platform @DataDog/apm-onboarding /pkg/clusteragent/orchestrator/ @DataDog/container-app -/pkg/clusteragent/telemetry/ @DataDog/telemetry-and-analytics +/pkg/clusteragent/telemetry/ @DataDog/apm-trace-storage /pkg/collector/ @DataDog/agent-metrics-logs /pkg/collector/corechecks/cluster/ @DataDog/container-integrations /pkg/collector/corechecks/cluster/orchestrator @DataDog/container-app @@ -351,9 +395,10 @@ /pkg/collector/corechecks/ebpf/ebpf* @DataDog/ebpf-platform /pkg/collector/corechecks/ebpf/probe/ebpfcheck/ @DataDog/ebpf-platform /pkg/collector/corechecks/ebpf/c/runtime/ebpf* @DataDog/ebpf-platform -/pkg/collector/corechecks/embed/ @Datadog/agent-devx-infra -/pkg/collector/corechecks/embed/apm/ @Datadog/agent-devx-infra @DataDog/agent-apm -/pkg/collector/corechecks/embed/process/ @Datadog/agent-devx-infra @DataDog/processes +/pkg/collector/corechecks/embed/ @Datadog/agent-delivery +/pkg/collector/corechecks/embed/apm/ @DataDog/agent-apm +/pkg/collector/corechecks/embed/process/ @DataDog/processes +/pkg/collector/corechecks/gpu/ @DataDog/ebpf-platform /pkg/collector/corechecks/network-devices/ @DataDog/network-device-monitoring /pkg/collector/corechecks/orchestrator/ @DataDog/container-app /pkg/collector/corechecks/kubernetes/ @DataDog/container-integrations @@ -399,7 +444,7 @@ /pkg/flare/*_windows_test.go @Datadog/windows-agent /pkg/fleet/ @DataDog/fleet @DataDog/windows-agent /pkg/otlp/ @DataDog/opentelemetry -/pkg/otlp/*_serverless*.go @DataDog/serverless +/pkg/otlp/*_serverless*.go @DataDog/serverless @Datadog/serverless-aws /pkg/otlp/*_not_serverless*.go @DataDog/opentelemetry /pkg/pidfile/ @DataDog/agent-shared-components /pkg/persistentcache/ @DataDog/agent-metrics-logs @@ -459,6 +504,9 @@ /pkg/process/metadata/parser/*windows* @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks @DataDog/windows-kernel-integrations /pkg/process/monitor/ @DataDog/universal-service-monitoring /pkg/process/net/ @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks +/pkg/process/net/common_windows.go @DataDog/windows-agent +/pkg/process/net/windows_pipe.go @DataDog/windows-kernel-integrations +/pkg/process/net/windows_pipe_testutil.go @DataDog/windows-kernel-integrations /pkg/proto/datadog/remoteconfig/ @DataDog/remote-config /pkg/proto/pbgo/ # do not notify anyone /pkg/proto/pbgo/trace @DataDog/agent-apm @@ -565,13 +613,10 @@ /test/ @DataDog/agent-devx-loops /test/benchmarks/ @DataDog/agent-metrics-logs /test/benchmarks/kubernetes_state/ @DataDog/container-integrations -/test/e2e/ @DataDog/container-integrations @DataDog/agent-security -/test/e2e/cws-tests/ @DataDog/agent-security -/test/e2e/argo-workflows/otlp-workflow.yaml @DataDog/opentelemetry -/test/e2e/containers/otlp_sender/ @DataDog/opentelemetry /test/integration/ @DataDog/container-integrations -/test/integration/serverless @DataDog/serverless -/test/integration/serverless_perf @DataDog/serverless +/test/integration/docker/otel_agent_build_tests.py @DataDog/opentelemetry +/test/integration/serverless @DataDog/serverless @Datadog/serverless-aws +/test/integration/serverless_perf @DataDog/serverless @Datadog/serverless-aws /test/kitchen/ @DataDog/agent-devx-loops /test/kitchen/test-definitions/ @DataDog/agent-delivery /test/kitchen/test/integration/ @DataDog/agent-delivery @@ -596,6 +641,7 @@ /test/fakeintake/aggregator/servicediscoveryAggregator.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring /test/fakeintake/aggregator/servicediscoveryAggregator_test.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring /test/new-e2e/ @DataDog/agent-e2e-testing @DataDog/agent-devx-loops +/test/new-e2e/pkg/components/datadog-installer @DataDog/windows-agent /test/new-e2e/test-infra-definition @DataDog/agent-devx-loops /test/new-e2e/system-probe @DataDog/ebpf-platform /test/new-e2e/scenarios/system-probe @DataDog/ebpf-platform @@ -632,6 +678,7 @@ /tools/ebpf/ @DataDog/ebpf-platform /tools/gdb/ @DataDog/agent-shared-components /tools/go-update/ @DataDog/agent-shared-components +/tools/NamedPipeCmd/ @DataDog/windows-kernel-integrations /tools/retry_file_dump/ @DataDog/agent-metrics-logs /tools/windows/ @DataDog/windows-agent /tools/windows/DatadogAgentInstaller/WixSetup/localization-en-us.wxl @DataDog/windows-agent @DataDog/documentation diff --git a/.github/workflows/backport-pr.yml b/.github/workflows/backport-pr.yml index 3ee7eeeb276b4..950d3ff41ca6a 100644 --- a/.github/workflows/backport-pr.yml +++ b/.github/workflows/backport-pr.yml @@ -24,7 +24,7 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/create-github-app-token@31c86eb3b33c9b601a1f60f98dcbfd1d70f379b4 # v1.10.3 + - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 id: app-token with: app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }} diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 523018890c0f6..454fc14e335a1 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -54,7 +54,7 @@ jobs: ref: ${{ inputs.branch }} - name: Setup Python and pip - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: # use Python < 3.12 so that distutil is still available by default python-version: 3.11 diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index 085cc6378cc0f..c1a4cff2baad0 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -19,7 +19,7 @@ jobs: with: ref: ${{ github.head_ref }} - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 748cd3e5aaeba..1c600968af15f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,9 +28,9 @@ jobs: fetch-depth: 0 - name: Setup Python3 - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: - python-version: "3.11.8" + python-version: "3.12.6" cache: "pip" - run: pip3 install -r requirements.txt diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index eec8e650f45ae..bd40afbf6e315 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -24,7 +24,7 @@ jobs: sparse-checkout: 'tasks' - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: "pip" @@ -64,7 +64,7 @@ jobs: fetch-depth: 0 - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index 4fc749d9b280c..4f674987ecf05 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -24,7 +24,7 @@ jobs: with: ref: ${{ github.head_ref }} - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index f335b2c5d8dde..0faf549f36d4c 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -64,7 +64,7 @@ jobs: sparse-checkout: ${{ matrix.cone }} - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: 'pip' @@ -103,7 +103,7 @@ jobs: ref: ${{ inputs.base_branch || 'main' }} - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: 'pip' diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 4ee67f94fadea..342987ab29ae8 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: '3.12' diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index d4092f41492ae..fd24b6828dbe1 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -22,7 +22,7 @@ jobs: ref: main fetch-depth: 0 - name: Setup python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: 'pip' @@ -31,7 +31,6 @@ jobs: run: pip install -r requirements.txt -r tasks/requirements.txt - name: Set label on external contributor PRs run: | - gh issue edit "$NUMBER" inv -e github.handle-community-pr --repo="$GH_REPO" --pr-id="$NUMBER" --labels="$LABELS" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index d90caf056fb82..a01b9d29ad803 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -30,7 +30,7 @@ jobs: with: go-version-file: ".go-version" - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index 1f0601757941f..29f35d223e758 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: Setup python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: 'pip' @@ -43,7 +43,7 @@ jobs: with: fetch-depth: 0 - name: Setup python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: 3.11 cache: 'pip' diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index dfbb06e3e6d3e..6d3c9280bbb91 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -11,6 +11,8 @@ permissions: {} jobs: comment: runs-on: ubuntu-latest + permissions: + pull-requests: write # Add comment to PR steps: - name: Checkout datadog-agent repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 diff --git a/.github/workflows/slapr.yml b/.github/workflows/slapr.yml deleted file mode 100644 index e88d67945b5a2..0000000000000 --- a/.github/workflows/slapr.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Disable Slapr until https://github.com/DataDog/slapr/issues/45 gets fixed. -# --- -# name: Slack emoji PR updates -# on: -# pull_request_review: -# types: [submitted] -# pull_request: -# types: [closed] -# -# permissions: {} -# -# jobs: -# run_slapr_datadog_agent: -# runs-on: ubuntu-latest -# strategy: -# matrix: -# channel_variables: -# - SLACK_AGENT_PLATFORM_REVIEWS_ID -# - SLACK_OPENTELEMETRY_ID -# - SLACK_AGENT_METRICS_LOGS_ID -# - SLACK_CONTAINER_INTEGRATIONS_ID -# - SLACK_EBPF_PLATFORM_ID -# steps: -# - uses: DataDog/slapr@master -# env: -# GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" -# SLACK_CHANNEL_ID: "${{ secrets[matrix.channel_variables] }}" -# SLACK_API_TOKEN: "${{ secrets.SLACK_API_TOKEN }}" -# SLAPR_BOT_USER_ID: "${{ secrets.SLAPR_BOT_USER_ID }}" -# SLAPR_EMOJI_REVIEW_STARTED: "review_started" -# SLAPR_EMOJI_APPROVED: "gh-approved" -# SLAPR_EMOJI_CHANGES_REQUESTED: "request-changes" -# SLAPR_EMOJI_MERGED: "merged" -# SLAPR_EMOJI_CLOSED: "closed" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc965a2f0a047..2ac6c19deb885 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,16 @@ --- include: - - .gitlab/.pre/cancel-prev-pipelines.yml - - .gitlab/.pre/test_gitlab_configuration.yml + - .gitlab/.pre/include.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml - .gitlab/choco_deploy/choco_deploy.yml + - .gitlab/powershell_script_signing/powershell_script_signing.yml + - .gitlab/powershell_script_deploy/powershell_script_deploy.yml - .gitlab/common/shared.yml - .gitlab/common/skip_ci_check.yml - - .gitlab/common/pr_commenter.yml - .gitlab/common/test_infra_version.yml - .gitlab/container_build/include.yml - .gitlab/container_scan/container_scan.yml @@ -22,7 +22,6 @@ include: - .gitlab/deps_fetch/deps_fetch.yml - .gitlab/dev_container_deploy/include.yml - .gitlab/e2e/e2e.yml - - .gitlab/e2e_k8s/e2e_k8s.yml - .gitlab/e2e_install_packages/include.yml - .gitlab/e2e_pre_test/e2e_pre_test.yml - .gitlab/functional_test/include.yml @@ -35,6 +34,7 @@ include: - .gitlab/kitchen_cleanup/include.yml - .gitlab/kitchen_deploy/kitchen_deploy.yml - .gitlab/kitchen_testing/include.yml + - .gitlab/lint/include.yml - .gitlab/maintenance_jobs/include.yml - .gitlab/notify/notify.yml - .gitlab/package_build/include.yml @@ -65,6 +65,7 @@ stages: - maintenance_jobs - deps_build - deps_fetch + - lint - source_test - source_test_stats - software_composition_analysis @@ -89,9 +90,9 @@ stages: - deploy_packages - deploy_cws_instrumentation - deploy_dca + - choco_and_install_script_build - trigger_release - - choco_build - - choco_deploy + - choco_and_install_script_deploy - internal_image_deploy - install_script_testing - e2e_pre_test @@ -141,6 +142,7 @@ variables: WINDOWS_TESTING_S3_BUCKET_A6: pipelines/A6/$CI_PIPELINE_ID WINDOWS_TESTING_S3_BUCKET_A7: pipelines/A7/$CI_PIPELINE_ID WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds + WINDOWS_POWERSHELL_DIR: $CI_PROJECT_DIR/signed_scripts DEB_RPM_TESTING_BUCKET_BRANCH: testing # branch of the DEB_TESTING_S3_BUCKET and RPM_TESTING_S3_BUCKET repos to release to, 'testing' S3_CP_OPTIONS: --no-progress --region us-east-1 --sse AES256 S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS @@ -169,15 +171,15 @@ variables: # To use images from datadog-agent-buildimages dev branches, set the corresponding # SUFFIX variable to _test_only DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BUILDIMAGES: v44534774-f5cc3e24 + DATADOG_AGENT_BUILDIMAGES: v45979046-89a73a40 DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_WINBUILDIMAGES: v44534774-f5cc3e24 + DATADOG_AGENT_WINBUILDIMAGES: v45979046-89a73a40 DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_ARMBUILDIMAGES: v44534774-f5cc3e24 + DATADOG_AGENT_ARMBUILDIMAGES: v45979046-89a73a40 DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v44534774-f5cc3e24 + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v45979046-89a73a40 DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v44534774-f5cc3e24 + DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v45979046-89a73a40 DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded DEB_GPG_KEY_ID: c0962c7d @@ -187,15 +189,15 @@ variables: DOCKER_REGISTRY_URL: docker.io KITCHEN_INFRASTRUCTURE_FLAKES_RETRY: 2 CLANG_LLVM_VER: 12.0.1 - KERNEL_MATRIX_TESTING_X86_AMI_ID: "ami-0c54d42f8f4180b0c" - KERNEL_MATRIX_TESTING_ARM_AMI_ID: "ami-021f04c00ecfa8590" + KERNEL_MATRIX_TESTING_X86_AMI_ID: "ami-013ea43cbe85107c0" + KERNEL_MATRIX_TESTING_ARM_AMI_ID: "ami-02fbd245475d6a63d" RUN_E2E_TESTS: "auto" # Should be "off", "auto" or "on" it will change the trigger condition for new-e2e tests on branch != main RUN_KMT_TESTS: "auto" # Should be "auto" or "on". "on" forces all Kernel Matrix Testing jobs to run. RUN_UNIT_TESTS: "auto" # Should be "auto", "on", "off" it will change the trigger condition for unit tests on branch != main # skip known flaky tests by default GO_TEST_SKIP_FLAKE: "true" - # List of parameters retrieved from AWS SSM + # Start aws ssm variables # They must be defined as environment variables in the GitLab CI/CD settings, to ease rotation if needed AGENT_QA_PROFILE: ci.datadog-agent.agent-qa-profile # agent-devx-infra API_KEY_ORG2: ci.datadog-agent.datadog_api_key_org2 # agent-devx-infra @@ -251,6 +253,15 @@ variables: SSH_PUBLIC_KEY_RSA: ci.datadog-agent.ssh_public_key_rsa # agent-devx-loops VCPKG_BLOB_SAS_URL: ci.datadog-agent-buildimages.vcpkg_blob_sas_url # windows-agent WINGET_PAT: ci.datadog-agent.winget_pat # windows-agent + # End aws ssm variables + # Start vault variables + # End vault variables + + ATLASSIAN_WRITE: atlassian-write + AGENT_GITHUB_APP: agent-github-app + MACOS_GITHUB_APP_1: macos-github-app-one + MACOS_GITHUB_APP_2: macos-github-app-two + DOCKER_REGISTRY_RO: dockerhub-readonly DD_PKG_VERSION: "latest" @@ -272,12 +283,6 @@ variables: .if_release_branch: &if_release_branch if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/ -.if_version_6: &if_version_6 - if: $RELEASE_VERSION_6 != "" - -.if_not_version_6: &if_not_version_6 - if: $RELEASE_VERSION_6 == "" - .if_version_7: &if_version_7 if: $RELEASE_VERSION_7 != "" @@ -416,10 +421,8 @@ workflow: - when: manual allow_failure: true -.except_no_a6_or_no_a7: +.except_no_a7: - !reference [.except_mergequeue] - - <<: *if_not_version_6 - when: never - <<: *if_not_version_7 when: never @@ -743,6 +746,8 @@ workflow: - pkg/network/**/* - pkg/process/monitor/* - pkg/util/kernel/**/* + - pkg/dynamicinstrumentation/**/* + - pkg/gpu/**/* - .gitlab/kernel_matrix_testing/system_probe.yml - .gitlab/kernel_matrix_testing/common.yml - .gitlab/source_test/ebpf.yml @@ -794,6 +799,7 @@ workflow: paths: - test/new-e2e/pkg/**/* - test/new-e2e/go.mod + - flakes.yaml compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 .on_e2e_or_windows_installer_changes: @@ -1032,6 +1038,15 @@ workflow: - when: manual allow_failure: true +.on_cspm_or_e2e_changes: + - !reference [.on_e2e_main_release_or_rc] + - changes: + paths: + - pkg/security/**/* + - test/new-e2e/tests/cspm/**/* #TODO: Add other paths that should trigger the execution of CSPM e2e tests + compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 + when: on_success + .on_windows_systemprobe_or_e2e_changes: - !reference [.on_e2e_main_release_or_rc] - changes: @@ -1143,12 +1158,17 @@ workflow: COVERAGE_CACHE_FLAG: "--pull-coverage-cache" .on_gitlab_changes: - - !reference [.except_mergequeue] - - changes: - paths: - - .gitlab-ci.yml - - .gitlab/**/* - compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 + changes: + paths: + - .gitlab-ci.yml + - .gitlab/**/* + - .gitlab/**/.* + compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 + +.on_gitlab_changes_or_mergequeue_or_main: + - !reference [.on_gitlab_changes] + - !reference [.on_mergequeue] + - !reference [.on_main] .on_invoke_tasks_changes: - <<: *if_main_branch diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml new file mode 100644 index 0000000000000..4848af1e13d53 --- /dev/null +++ b/.gitlab/.ci-linters.yml @@ -0,0 +1,111 @@ +--- +# Configuration for CI related linters + +# Lists stages and jobs that are not allowed to have both 'needs' and 'rules' keywords +needs-rules: + allowed-stages: + - .pre + - lint + - notify + - deploy_containers + + allowed-jobs: + - build_clang_arm64 + - build_clang_x64 + - build_dogstatsd_static-binary_x64 + - build_processed_btfhub_archive + - check_already_deployed_version_7 + - cleanup_kitchen_functional_test + - deploy_containers-cws-instrumentation-final-versioned + - deploy_containers-cws-instrumentation-latest + - deploy_containers-cws-instrumentation-rc-mutable + - deploy_containers-cws-instrumentation-rc-versioned + - dogstatsd_x64_size_test + - go_mod_tidy_check + - kitchen_cleanup_azure-a7 + - lint_flavor_dogstatsd_linux-x64 + - lint_flavor_heroku_linux-x64 + - lint_flavor_iot_linux-x64 + - lint_linux-arm64 + - lint_linux-x64 + - lint_macos_gitlab_amd64 + - periodic_kitchen_cleanup_azure + - periodic_kitchen_cleanup_ec2 + - periodic_kitchen_cleanup_s3 + - publish_winget_7_x64 + - revert_latest_7 + - security_go_generate_check + - setup_agent_version + - tests_ebpf_arm64 + - tests_ebpf_x64 + - tests_windows_secagent_x64 + - tests_windows_sysprobe_x64 + - trigger_auto_staging_release + - trigger_manual_prod_release + +# Lists jobs that are allowed to not be within JOBOWNERS +job-owners: + allowed-jobs: + - benchmark + - bootstrapper_build + - build_dogstatsd-binary_arm64 + - build_dogstatsd-binary_x64 + - build_dogstatsd_static-binary_arm64 + - build_dogstatsd_static-binary_x64 + - build_fakeintake + - build_iot_agent-binary_arm64 + - build_iot_agent-binary_x64 + - build_otel_agent_binary_arm64 + - build_otel_agent_binary_x64 + - cancel-prev-pipelines + - check_pkg_size-amd64-a7 + - check_pkg_size-arm64-a7 + - close_failing_tests_stale_issues + - compute_gitlab_ci_config + - deploy_cluster_agent_cloudfoundry + - do-not-merge + - dogstatsd_x64_size_test + - generate-flakes-finder-pipeline + - github_rate_limit_info + - go_deps + - go_e2e_deps + - go_mod_tidy_check + - go_tools_deps + - installer-amd64 + - installer-amd64-oci + - installer-arm64 + - installer-arm64-oci + - integration_tests_otel + - invoke_unit_tests + - kitchen_cleanup_azure-a7 + - kitchen_invoke_unit_tests + - new-e2e-cspm + - new-e2e-otel + - new-e2e-package-signing-debian-a7-x86_64 + - new-e2e-package-signing-suse-a7-x86_64 + - new-e2e-unit-tests + - ot_agent_deb-arm64-a7 + - ot_agent_deb-x64-a7 + - periodic_kitchen_cleanup_azure + - periodic_kitchen_cleanup_ec2 + - periodic_kitchen_cleanup_s3 + - publish_choco_7_x64 + - publish_fakeintake + - publish_fakeintake_latest + - rc_kubernetes_deploy + - revert_latest_7 + - send_pipeline_stats + - send_pkg_size + - setup_agent_version + - single-machine-performance-regression_detector + - single-machine-performance-regression_detector-pr-comment + - test_install_script + - trace-agent-v04-2cpus-normal_load-fixed_sps + - trace-agent-v04-2cpus-stress_load-fixed_sps + - trace-agent-v05-2cpus-normal_load-fixed_sps + - trace-agent-v05-2cpus-stress_load-fixed_sps + - trigger-flakes-finder + - unit_tests_arm64_junit_upload + - unit_tests_notify + - update_rc_build_links + - validate_modules diff --git a/.gitlab/.pre/create_release_qa_cards.yml b/.gitlab/.pre/create_release_qa_cards.yml new file mode 100644 index 0000000000000..d22c256bb3609 --- /dev/null +++ b/.gitlab/.pre/create_release_qa_cards.yml @@ -0,0 +1,14 @@ +--- +create_release_qa_cards: + stage: .pre + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + rules: + - !reference [.on_deploy_rc] + script: + - !reference [.setup_agent_github_app] + - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD + - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME + - pip install ddqa + - inv release.create-qa-cards -t ${CI_COMMIT_REF_NAME} + allow_failure: true diff --git a/.gitlab/.pre/gitlab_configuration.yml b/.gitlab/.pre/gitlab_configuration.yml new file mode 100644 index 0000000000000..de70688431253 --- /dev/null +++ b/.gitlab/.pre/gitlab_configuration.yml @@ -0,0 +1,82 @@ +test_gitlab_configuration: + stage: .pre + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + rules: + - !reference [.except_mergequeue] + - !reference [.on_gitlab_changes] + script: + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - inv -e linter.gitlab-ci + - inv -e linter.job-change-path + - inv -e linter.gitlab-change-paths + - inv -e linter.ssm-parameters + +test_gitlab_compare_to: + stage: .pre + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + rules: + - !reference [.except_mergequeue] + - !reference [.on_gitlab_changes] + script: + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - !reference [.setup_agent_github_app] + - pip install -r tasks/requirements.txt + - inv pipeline.compare-to-itself + +.compute_gitlab_ci_config_rules: + - if: $CI_PIPELINE_SOURCE != "push" + when: never + +# Computes and uploads the GitLab CI configuration diff as an artifact +compute_gitlab_ci_config: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + stage: .pre + needs: [] + tags: ["arch:arm64"] + rules: + - !reference [.compute_gitlab_ci_config_rules] + - when: on_success + before_script: + # Get main history + - git fetch origin main + - git checkout main + - git checkout - + script: + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - mkdir -p artifacts + - inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml + artifacts: + when: always + paths: + - artifacts/ + expire_in: 1 day + +lint_gitlab_ci: + stage: .pre + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + needs: [compute_gitlab_ci_config] + rules: + - !reference [.compute_gitlab_ci_config_rules] + - !reference [.on_gitlab_changes_or_mergequeue_or_main] + script: + - status=0 + - inv -e linter.gitlab-ci-jobs-needs-rules --config-file artifacts/after.gitlab-ci.yml || status=1 + - inv -e linter.gitlab-ci-jobs-owners --config-file artifacts/after.gitlab-ci.yml || status=1 + - | + if [ $status != 0 ]; then + echo "At least one linter failed, exiting..." >& 2 + exit $status + fi + +lint_gitlab_ci_jobs_codeowners: + stage: .pre + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:arm64"] + needs: [] + rules: + - !reference [.on_gitlab_changes] + script: + - inv -e linter.gitlab-ci-jobs-codeowners --all-files diff --git a/.gitlab/.pre/include.yml b/.gitlab/.pre/include.yml new file mode 100644 index 0000000000000..ac759921018b0 --- /dev/null +++ b/.gitlab/.pre/include.yml @@ -0,0 +1,7 @@ +--- +# Define preliminary jobs, which can start as soon as possible + +include: + - .gitlab/.pre/cancel-prev-pipelines.yml + - .gitlab/.pre/create_release_qa_cards.yml + - .gitlab/.pre/gitlab_configuration.yml diff --git a/.gitlab/.pre/test_gitlab_configuration.yml b/.gitlab/.pre/test_gitlab_configuration.yml deleted file mode 100644 index 1de3d176087d5..0000000000000 --- a/.gitlab/.pre/test_gitlab_configuration.yml +++ /dev/null @@ -1,24 +0,0 @@ -test_gitlab_configuration: - stage: .pre - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: - - !reference [.on_gitlab_changes] - script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN - - inv -e linter.gitlab-ci - - inv -e linter.job-change-path - - inv -e linter.gitlab-change-paths - - inv -e linter.ssm-parameters - -test_gitlab_compare_to: - stage: .pre - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - rules: - - !reference [.on_gitlab_changes] - script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN - - !reference [.setup_agent_github_app] - - pip install -r tasks/requirements.txt - - inv pipeline.compare-to-itself diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index b4cd8f2bd1d99..f8f9ab4fa4766 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -1,5 +1,14 @@ * @DataDog/agent-devx-infra +# CI / DevX +test_gitlab* @DataDog/agent-devx-infra +invoke_unit_tests @DataDog/agent-devx-infra +skip-ci-check @DataDog/agent-devx-infra +slack_teams_channels_check @DataDog/agent-devx-infra +lint* @DataDog/agent-devx-infra +notify* @DataDog/agent-devx-infra +create_release_qa_cards @DataDog/agent-devx-infra + # Deps build build_clang_* @DataDog/ebpf-platform build_processed_btfhub_archive @DataDog/ebpf-platform @@ -42,6 +51,7 @@ iot-agent-* @DataDog/agent-delivery agent_suse* @DataDog/agent-delivery windows-installer* @DataDog/windows-agent windows_* @DataDog/windows-agent +powershell_script_signing @DataDog/windows-agent # Packaging agent_deb* @DataDog/agent-delivery @@ -90,7 +100,6 @@ qa_cws_instrumentation* @DataDog/agent-devx-loops qa_dca* @DataDog/agent-devx-loops qa_dogstatsd* @DataDog/agent-devx-loops - # Internal image deploy docker_trigger_internal* @DataDog/container-integrations docker_trigger_cluster_agent_internal* @DataDog/container-integrations @@ -105,6 +114,8 @@ deploy_installer* @DataDog/agent-delivery deploy_packages* @DataDog/agent-delivery deploy_staging* @DataDog/agent-delivery publish_winget* @DataDog/windows-agent +powershell_script_deploy @DataDog/windows-agent +windows_bootstrapper_deploy @DataDog/windows-agent qa_*_oci @DataDog/agent-delivery # Deploy containers @@ -119,6 +130,8 @@ trigger_auto_staging_release @DataDog/agent-delivery # Integration test integration_tests_windows* @DataDog/windows-agent +integration_tests_otel @DataDog/opentelemetry +docker_image_build_otel @DataDog/opentelemetry # Functional test kitchen_*_system_probe_windows* @DataDog/windows-kernel-integrations @@ -140,18 +153,21 @@ new-e2e-agent-shared-components* @DataDog/agent-shared-components new-e2e-language-detection* @DataDog/processes new-e2e-process* @DataDog/processes new-e2e-agent-platform* @DataDog/agent-delivery +new-e2e-platform-integrations* @DataDog/agent-delivery @DataDog/platform-integrations new-e2e-aml* @DataDog/agent-metrics-logs new-e2e-apm* @DataDog/agent-apm new-e2e-discovery* @Datadog/universal-service-monitoring new-e2e-ndm* @DataDog/network-device-monitoring new-e2e-npm* @DataDog/Networks new-e2e-cws* @DataDog/agent-security -new-e2e-windows-agent* @DataDog/windows-agent new-e2e-orchestrator* @DataDog/container-app e2e_pre_test* @DataDog/agent-devx-loops new-e2e-remote-config* @DataDog/remote-config new-e2e-installer* @DataDog/fleet -new-e2e-windows-service-test @DataDog/windows-agent +new-e2e-installer-windows @DataDog/windows-agent +new-e2e-windows* @DataDog/windows-agent +new-e2e-windows-systemprobe @DataDog/windows-kernel-integrations +new-e2e-windows-security-agent @DataDog/windows-kernel-integrations new-e2e_windows_powershell_module_test @DataDog/windows-kernel-integrations # Kernel matrix testing diff --git a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml index 873a471a3b85a..0a3b9802eb47b 100644 --- a/.gitlab/binary_build/cluster_agent_cloudfoundry.yml +++ b/.gitlab/binary_build/cluster_agent_cloudfoundry.yml @@ -21,5 +21,5 @@ cluster_agent_cloudfoundry-build_amd64: - inv -e cluster-agent-cloudfoundry.build - cd $CI_PROJECT_DIR/$CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR - mkdir -p $OMNIBUS_PACKAGE_DIR - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) + - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $? - tar cf $OMNIBUS_PACKAGE_DIR/datadog-cluster-agent-cloudfoundry-$PACKAGE_VERSION-$ARCH.tar.xz datadog-cluster-agent-cloudfoundry diff --git a/.gitlab/check_deploy/check_deploy.yml b/.gitlab/check_deploy/check_deploy.yml index 6ac61ba1dba62..438799d573dfb 100644 --- a/.gitlab/check_deploy/check_deploy.yml +++ b/.gitlab/check_deploy/check_deploy.yml @@ -1,24 +1,12 @@ --- # check_deploy stage -# Contains jobs which which check if the version of the Agent built in the pipeline -# alteady exists (in the apt staging repo), or if the release branch is "none". +# Contains jobs which check if the version of the Agent built in the pipeline +# already exists (in the apt staging repo), or if the release branch is "none". # In both cases, these jobs fail in order to prevent the pipeline from deploying artifacts. # Check that the current version hasn't already been deployed (we don't want to # overwrite a public package). To update an erroneous package, first remove it # from our S3 bucket. -check_already_deployed_version_6: - rules: !reference [.on_deploy] - stage: check_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - dependencies: ["agent_deb-x64-a6", "agent_deb-arm64-a6"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_6*_amd64.deb - - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh datadog-agent_6*_arm64.deb - check_already_deployed_version_7: rules: !reference [.on_deploy] stage: check_deploy diff --git a/.gitlab/choco_build/choco_build.yml b/.gitlab/choco_build/choco_build.yml index d14e832b9b15b..61be7eb6e75bd 100644 --- a/.gitlab/choco_build/choco_build.yml +++ b/.gitlab/choco_build/choco_build.yml @@ -6,7 +6,7 @@ windows_choco_offline_7_x64: rules: !reference [.manual] - stage: choco_build + stage: choco_and_install_script_build tags: ["runner:windows-docker", "windowsversion:1809"] needs: ["windows_msi_and_bosh_zip_x64-a7"] variables: @@ -28,7 +28,7 @@ windows_choco_offline_7_x64: windows_choco_online_7_x64: rules: !reference [.on_deploy_stable_or_beta_repo_branch_manual] - stage: choco_build + stage: choco_and_install_script_build tags: ["runner:windows-docker", "windowsversion:1809"] needs: ["deploy_staging_windows_tags-7"] variables: @@ -42,6 +42,8 @@ windows_choco_online_7_x64: - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\chocopack.bat online - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - copy build-out\*.nupkg omnibus\pkg + - $CopyNupkgToS3 = "$S3_CP_CMD --recursive --exclude '*' --include '*.nupkg' build-out $S3_RELEASE_ARTIFACTS_URI/choco/nupkg" + - Invoke-Expression $CopyNupkgToS3 artifacts: expire_in: 2 weeks paths: diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml index 715387a08f961..56e66603fe9af 100644 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ b/.gitlab/choco_deploy/choco_deploy.yml @@ -4,14 +4,14 @@ publish_choco_7_x64: rules: !reference [.on_deploy_stable_or_beta_repo_branch_manual] - stage: choco_deploy + stage: choco_and_install_script_deploy tags: ["runner:windows-docker", "windowsversion:1809"] needs: ["windows_choco_online_7_x64"] variables: ARCH: "x64" before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:CHOCOLATEY_API_KEY" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $chocolateyApiKey=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/.gitlab/common/macos.yml b/.gitlab/common/macos.yml new file mode 100644 index 0000000000000..580384caaf894 --- /dev/null +++ b/.gitlab/common/macos.yml @@ -0,0 +1,32 @@ +--- +# This is the scripts to be executed on the Gitlab macOS runners before every job. +# We don't have virtualization now so we need to clean the environment and install the proper dependencies before every job. +.macos_gitlab: + variables: + PYTHON_RUNTIMES: "3" + before_script: + # Selecting the current Go version + - | + eval $(gimme $(cat .go-version)) + export GOPATH=$GOROOT + echo "Don't forget to regularly delete unused versions. Here are the installed versions and their memory usage on the runner:" + du -sh $HOME/.gimme/versions/* + # Remove the Python cache and env if the Python version changed + - | + PYTHON_REPO_VERSION=$(cat .python-version) + PYTHON_VERSION=$(python3 --version | awk '{print $2}' | sed 's/\.[0-9]*$//') + if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then + echo "Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version. Cleaning the environment." + pyenv uninstall -f datadog-agent + echo "Installing Python $PYTHON_REPO_VERSION..." + pyenv virtualenv 3.11.8 datadog-agent + pyenv activate datadog-agent + else + echo "Python current version $PYTHON_VERSION is the same as .python-version. Keeping the existing environment." + fi + # Installing the job dependencies + - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-github.txt + - pyenv rehash + - inv -e rtloader.make --python-runtimes $PYTHON_RUNTIMES + - inv -e rtloader.install + - inv -e install-tools diff --git a/.gitlab/common/pr_commenter.yml b/.gitlab/common/pr_commenter.yml deleted file mode 100644 index 75728807ae0c4..0000000000000 --- a/.gitlab/common/pr_commenter.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -.install_pr_commenter: - # Add janky means of installing PR commenter borrowed from - # https://github.com/DataDog/dogweb/blob/45d7fcf035d0d515ebd901919099d4c8bfa82829/docker/docker-builder/Dockerfile#L69-L77 - - apt-get update - - apt-get install -y curl - - curl -OL https://s3.amazonaws.com/dd-package-public/dd-package.deb - - dpkg -i dd-package.deb - - rm dd-package.deb - - apt-get update - - dd-package --bucket binaries.ddbuild.io --package devtools/dd-package-dev --distribution "20.04" - # Kludge from https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4645#note_287636439 to avoid - # doubled output - - echo "" - - "####################### NOTE TO MAINTAINERS #####################################" - - "# Ignore bdist_wheel build error raised when installing 'devtools/pr-commenter' #" - - "# This error is known behavior, and you should see a successful install #" - - "# beneath the error message in the GitLab CI logs #" - - "#################################################################################" - - echo "" - # Pinned version of pr-commenter taken from https://github.com/DataDog/dogweb/blob/b249c0815f70e71e07846f5e42fb6504c17a7c3e/tasks/gitlab/k8s-diff-helper.sh#L22 - - dd-package --bucket binaries.ddbuild.io --package devtools/pr-commenter --distribution "20.04" --version "16841676-368d2e2d" diff --git a/.gitlab/common/shared.yml b/.gitlab/common/shared.yml index 8a16870d9fac2..b00293abc750e 100644 --- a/.gitlab/common/shared.yml +++ b/.gitlab/common/shared.yml @@ -30,21 +30,21 @@ # This balances the requests made to GitHub between the two apps we have set up. - | if [[ "$(( RANDOM % 2 ))" == "1" ]]; then - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 + GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID + GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID echo "Using GitHub App instance 1" else - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY_2) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID_2) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID_2) || exit $?; export GITHUB_INSTALLATION_ID + GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 + GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID + GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID echo "Using GitHub App instance 2" fi .setup_agent_github_app: - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_GITHUB_APP installation_id) || exit $?; export GITHUB_INSTALLATION_ID - echo "Using agent GitHub App" # Install `dd-pkg` and lint packages produced by Omnibus, supports only deb and rpm packages diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index bf547355d351a..31e364287c40c 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -4,4 +4,4 @@ variables: # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 2a59d2442896 + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 08b9d8e048c1 diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index b6c028e032a01..4b0ff59a4369b 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -13,8 +13,8 @@ fi - TARGET_TAG=${IMAGE}${ECR_RELEASE_SUFFIX}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}$TAG_SUFFIX-$ARCH # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_LOGIN) || exit $? - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_PWD | docker login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" + - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO user) || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO token | docker login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi # Build image, use target none label to avoid replication - docker buildx build --no-cache --push --pull --platform linux/$ARCH --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} $BUILD_ARG --build-arg DD_GIT_REPOSITORY_URL=https://github.com/DataDog/datadog-agent --build-arg DD_GIT_COMMIT_SHA=${CI_COMMIT_SHA} --file $BUILD_CONTEXT/Dockerfile --tag ${TARGET_TAG} --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label "org.opencontainers.image.authors=Datadog " --label "org.opencontainers.image.source=https://github.com/DataDog/datadog-agent" --label "org.opencontainers.image.version=$(inv agent.version)" --label "org.opencontainers.image.revision=${CI_COMMIT_SHA}" --label "org.opencontainers.image.vendor=Datadog, Inc." --label "target=none" $BUILD_CONTEXT @@ -38,79 +38,6 @@ variables: ARCH: arm64 -# build agent6 py2 image -docker_build_agent6: - extends: .docker_build_job_definition_amd64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: registry.ddbuild.io/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - -docker_build_agent6_arm64: - extends: .docker_build_job_definition_arm64 - rules: !reference [.on_all_builds] - needs: - - job: agent_deb-arm64-a6 - artifacts: false - variables: - IMAGE: registry.ddbuild.io/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6 - BUILD_ARG: --target test --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*arm64.deb - -# build agent6 py2 jmx image -docker_build_agent6_jmx: - extends: .docker_build_job_definition_amd64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: registry.ddbuild.io/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - BUILD_ARTIFACT_GLOB: datadog-agent_6*_amd64.deb - TAG_SUFFIX: -6-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - -# build agent6 py2 jmx image -docker_build_agent6_jmx_arm64: - extends: .docker_build_job_definition_arm64 - rules: !reference [.on_all_builds] - needs: - - job: agent_deb-arm64-a6 - artifacts: false - variables: - IMAGE: registry.ddbuild.io/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - BUILD_ARTIFACT_GLOB: datadog-agent_6*arm64.deb - TAG_SUFFIX: -6-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=2 --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*arm64.deb - -# TESTING ONLY: This image is for internal testing purposes, not customer facing. -# build agent6 jmx unified image (including python3) -docker_build_agent6_py2py3_jmx: - extends: .docker_build_job_definition_amd64 - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - job: agent_deb-x64-a6 - artifacts: false - variables: - IMAGE: registry.ddbuild.io/ci/datadog-agent/agent - BUILD_CONTEXT: Dockerfiles/agent - TAG_SUFFIX: -6-py2py3-jmx - BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent_6*_amd64.deb - # build agent7 image docker_build_agent7: extends: .docker_build_job_definition_amd64 diff --git a/.gitlab/container_build/docker_windows.yml b/.gitlab/container_build/docker_windows.yml index af2a6a84bfde6..dada869f2a0d3 100644 --- a/.gitlab/container_build/docker_windows.yml +++ b/.gitlab/container_build/docker_windows.yml @@ -36,9 +36,9 @@ powershell -C C:\mnt\tools\ci\docker-login.ps1 - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - powershell -Command "docker build --no-cache --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} ${BUILD_ARG} --pull --file ${BUILD_CONTEXT}/windows/amd64/Dockerfile --tag ${TARGET_TAG} ${BUILD_CONTEXT}" + - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker build --no-cache --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} ${BUILD_ARG} --pull --file ${BUILD_CONTEXT}/windows/amd64/Dockerfile --tag ${TARGET_TAG} ${BUILD_CONTEXT}" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - docker push ${TARGET_TAG} + - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker push ${TARGET_TAG}" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - docker rmi ${TARGET_TAG} - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } @@ -55,24 +55,6 @@ AGENT_ZIP: "datadog-agent-7*-x86_64.zip" BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:lts-nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}" -.docker_build_agent6_windows_common: - extends: - - .docker_build_agent_windows_common - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["windows_msi_x64-a6", "build_windows_container_entrypoint"] - variables: - AGENT_ZIP: "datadog-agent-6*-x86_64.zip" - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:lts-nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}" - -.docker_build_agent6_windows_servercore_common: - extends: - - .docker_build_agent6_windows_common - variables: - BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:windowsservercore-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=core-${VARIANT}" - SERVERCORE: "-servercore" - .docker_build_agent7_windows_servercore_common: extends: - .docker_build_agent7_windows_common @@ -81,5 +63,4 @@ SERVERCORE: "-servercore" include: - - .gitlab/container_build/docker_windows_agent6.yml - .gitlab/container_build/docker_windows_agent7.yml diff --git a/.gitlab/container_build/docker_windows_agent6.yml b/.gitlab/container_build/docker_windows_agent6.yml deleted file mode 100644 index e2ef18af2bcc2..0000000000000 --- a/.gitlab/container_build/docker_windows_agent6.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Python 2 does not run on nanoserver -docker_build_agent6_windows1809_core: - extends: - - .docker_build_agent6_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:1809"] - variables: - VARIANT: 1809 - TAG_SUFFIX: -6 - WITH_JMX: "false" - -docker_build_agent6_windows2022_core: - extends: - - .docker_build_agent6_windows_servercore_common - tags: ["runner:windows-docker", "windowsversion:2022"] - variables: - VARIANT: ltsc2022 - TAG_SUFFIX: -6 - WITH_JMX: "false" diff --git a/.gitlab/container_build/fakeintake.yml b/.gitlab/container_build/fakeintake.yml index a348a3c729cb4..e0c4ec8c30400 100644 --- a/.gitlab/container_build/fakeintake.yml +++ b/.gitlab/container_build/fakeintake.yml @@ -15,8 +15,8 @@ docker_build_fakeintake: BUILD_CONTEXT: . script: # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_LOGIN) || exit $? - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_PWD | docker login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" + - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO user) || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO token | docker login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi - docker buildx build --push --pull --platform ${PLATFORMS} --file ${DOCKERFILE} --tag ${TARGET} $BUILD_CONTEXT retry: 2 diff --git a/.gitlab/container_scan/container_scan.yml b/.gitlab/container_scan/container_scan.yml index cf8cc90908d61..650e16e1913ea 100644 --- a/.gitlab/container_scan/container_scan.yml +++ b/.gitlab/container_scan/container_scan.yml @@ -15,23 +15,7 @@ scan_nightly-dogstatsd: IMG_REGISTRIES: dev IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 IMG_DESTINATIONS: dogstatsd-scan:${BUCKET_BRANCH} - -scan_nightly-a6: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_deploy_nightly_repo_branch] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-scan:${BUCKET_BRANCH}-py2-jmx + timeout: 15m scan_nightly-a7: extends: .docker_publish_job_definition @@ -74,23 +58,6 @@ scan_master-dogstatsd: IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64 IMG_DESTINATIONS: dogstatsd-scan:master -scan_master-a6: - extends: .docker_publish_job_definition - stage: container_scan - rules: - !reference [.on_main] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-scan:master-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-scan:master-py2-jmx - scan_master-a7: extends: .docker_publish_job_definition stage: container_scan diff --git a/.gitlab/deploy_containers/deploy_containers_a7.yml b/.gitlab/deploy_containers/deploy_containers_a7.yml index d4b2ae4d6ad8f..05eb4b531708c 100644 --- a/.gitlab/deploy_containers/deploy_containers_a7.yml +++ b/.gitlab/deploy_containers/deploy_containers_a7.yml @@ -25,7 +25,7 @@ include: stage: deploy_containers dependencies: [] before_script: - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi - export IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7${JMX}-amd64,${IMG_BASE_SRC}-7${JMX}-arm64" - export IMG_WINDOWS_SOURCES="${IMG_BASE_SRC}-7${JMX}-win1809${FLAVOR}-amd64,${IMG_BASE_SRC}-7${JMX}-winltsc2022${FLAVOR}-amd64" @@ -75,7 +75,7 @@ deploy_containers-dogstatsd: !reference [.manual_on_deploy_auto_on_rc] dependencies: [] before_script: - - export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" + - VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $? - export IMG_SOURCES="${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64" - export IMG_DESTINATIONS="${DSD_REPOSITORY}:${VERSION}" @@ -98,7 +98,7 @@ deploy_containers-a7_internal-rc: deploy_containers-ot: extends: .deploy_containers-a7-base-ot before_script: - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi - export IMG_SOURCES="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta${JMX}-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta${JMX}-arm64" - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}-ot-beta${JMX}" parallel: diff --git a/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml b/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml index 2d7301d70bca4..330c60bb239ea 100644 --- a/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml +++ b/.gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml @@ -11,7 +11,7 @@ include: stage: deploy_cws_instrumentation dependencies: [] before_script: - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe)" || exit $?; fi - if [[ "$CWS_INSTRUMENTATION_REPOSITORY" == "" ]]; then export CWS_INSTRUMENTATION_REPOSITORY="cws-instrumentation"; fi - export IMG_BASE_SRC="${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" diff --git a/.gitlab/deploy_dca/deploy_dca.yml b/.gitlab/deploy_dca/deploy_dca.yml index 5065744f8e315..63ef1ed56d74c 100644 --- a/.gitlab/deploy_dca/deploy_dca.yml +++ b/.gitlab/deploy_dca/deploy_dca.yml @@ -15,7 +15,7 @@ include: - job: "docker_build_cluster_agent_arm64" artifacts: false before_script: - - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi + - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --major-version 7 --url-safe)" || exit $?; fi - if [[ "$CLUSTER_AGENT_REPOSITORY" == "" ]]; then export CLUSTER_AGENT_REPOSITORY="cluster-agent"; fi - export IMG_BASE_SRC="${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64" diff --git a/.gitlab/deploy_packages/deploy_common.yml b/.gitlab/deploy_packages/deploy_common.yml index 327f1272dff16..f82ee5b64a689 100644 --- a/.gitlab/deploy_packages/deploy_common.yml +++ b/.gitlab/deploy_packages/deploy_common.yml @@ -110,3 +110,15 @@ variables: ARTIFACTS_PREFIX: suse_ OMNIBUS_PACKAGE_DIR: $OMNIBUS_PACKAGE_DIR_SUSE + +deploy_installer_boostrapper: + rules: + !reference [.on_deploy_installer] + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + stage: deploy_packages + needs: ["bootstrapper_build"] + tags: ["arch:amd64"] + before_script: + - ls $OMNIBUS_PACKAGE_DIR + script: + - $S3_CP_CMD --recursive --exclude "*" --include "bootstrapper-*" "$OMNIBUS_PACKAGE_DIR" "${S3_RELEASE_INSTALLER_ARTIFACTS_URI}/bootstrapper/" diff --git a/.gitlab/deploy_packages/nix.yml b/.gitlab/deploy_packages/nix.yml index 4baead2992b04..bac097597a683 100644 --- a/.gitlab/deploy_packages/nix.yml +++ b/.gitlab/deploy_packages/nix.yml @@ -1,44 +1,4 @@ --- -# -# Agent v6 -# -deploy_packages_deb-x64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_deb-x64-a6 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_deb-arm64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_deb-arm64-a6 ] - variables: - PACKAGE_ARCH: arm64 - -deploy_packages_heroku_deb-x64-6: - extends: .deploy_packages_deb-6 - needs: [ agent_heroku_deb-x64-a6 ] - variables: - PACKAGE_ARCH: amd64 - -deploy_packages_rpm-x64-6: - extends: .deploy_packages_rpm-6 - needs: [ agent_rpm-x64-a6 ] - variables: - PACKAGE_ARCH: x86_64 - -deploy_packages_rpm-arm64-6: - extends: .deploy_packages_rpm-6 - needs: [ agent_rpm-arm64-a6 ] - variables: - PACKAGE_ARCH: aarch64 - -# NOTE: no SuSE ARM builds currently. -deploy_packages_suse_rpm-x64-6: - extends: .deploy_packages_suse_rpm-6 - needs: [ agent_suse-x64-a6 ] - variables: - PACKAGE_ARCH: x86_64 - # # Agent v7 # @@ -172,7 +132,7 @@ deploy_staging_dsd: needs: ["build_dogstatsd-binary_x64"] script: - $S3_CP_CMD $S3_ARTIFACTS_URI/dogstatsd/dogstatsd ./dogstatsd - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) + - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $? - $S3_CP_CMD ./dogstatsd $S3_DSD6_URI/linux/dogstatsd-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 # deploy iot-agent x64 binary to staging bucket @@ -185,7 +145,7 @@ deploy_staging_iot_agent: needs: ["build_iot_agent-binary_x64"] script: - $S3_CP_CMD $S3_ARTIFACTS_URI/iot/agent ./agent - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) + - PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7) || exit $? - $S3_CP_CMD ./agent $S3_DSD6_URI/linux/iot/agent-$PACKAGE_VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 # Datadog Installer diff --git a/.gitlab/deploy_packages/oci.yml b/.gitlab/deploy_packages/oci.yml index da2dd66b2e11e..b3e39afe86fb1 100644 --- a/.gitlab/deploy_packages/oci.yml +++ b/.gitlab/deploy_packages/oci.yml @@ -15,7 +15,8 @@ include: - ls $OMNIBUS_PACKAGE_DIR script: - set +x - - export VERSION=$(inv agent.version --url-safe)-1 + - !reference [.retrieve_linux_go_tools_deps] + - VERSION="$(inv agent.version --url-safe)-1" || exit $? - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - go env -w GOPRIVATE="github.com/DataDog/*" - export PATH=$PATH:$(go env GOPATH)/bin @@ -28,12 +29,12 @@ include: deploy_agent_oci: extends: ".deploy_packages_oci" - needs: [ "agent_oci" ] + needs: [ "agent_oci", "go_tools_deps"] variables: OCI_PRODUCT: "datadog-agent" deploy_installer_oci: extends: ".deploy_packages_oci" - needs: [ "installer_oci" ] + needs: [ "installer_oci", "go_tools_deps" ] variables: OCI_PRODUCT: "datadog-installer" diff --git a/.gitlab/deploy_packages/windows.yml b/.gitlab/deploy_packages/windows.yml index c119e50f48048..a5b10e40d712d 100644 --- a/.gitlab/deploy_packages/windows.yml +++ b/.gitlab/deploy_packages/windows.yml @@ -1,24 +1,4 @@ --- -# -# Agent v6 -# -deploy_packages_windows-x64-6: - rules: - !reference [.on_deploy] - stage: deploy_packages - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["windows_msi_x64-a6"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD - --recursive - --exclude "*" - --include "datadog-agent-6*.msi" - --include "datadog-agent-6*.debug.zip" - $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/msi/x86_64/ - # # Agent v7 # @@ -84,4 +64,5 @@ deploy_installer_packages_windows-x64: --include "datadog-installer-*-1-x86_64.msi" --include "datadog-installer-*-1-x86_64.debug.zip" --include "datadog-installer-*-1-x86_64.zip" + --include "datadog-installer-*-1-x86_64.exe" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_INSTALLER_ARTIFACTS_URI/msi/x86_64/ diff --git a/.gitlab/deploy_packages/winget.yml b/.gitlab/deploy_packages/winget.yml index a35239c948381..f28f946b1fb0c 100644 --- a/.gitlab/deploy_packages/winget.yml +++ b/.gitlab/deploy_packages/winget.yml @@ -11,7 +11,7 @@ publish_winget_7_x64: ARCH: "x64" before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:WINGET_PAT" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:WINGET_PAT" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $wingetPat=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/.gitlab/deps_fetch/deps_fetch.yml b/.gitlab/deps_fetch/deps_fetch.yml index 6defdd7d381e7..f44e3010236af 100644 --- a/.gitlab/deps_fetch/deps_fetch.yml +++ b/.gitlab/deps_fetch/deps_fetch.yml @@ -4,15 +4,15 @@ # to reuse them in further jobs that need them. .retrieve_linux_go_deps: - - mkdir -p $GOPATH/pkg/mod && tar xJf modcache.tar.xz -C $GOPATH/pkg/mod + - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache.tar.xz -C $GOPATH/pkg/mod/cache - rm -f modcache.tar.xz .retrieve_linux_go_tools_deps: - - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod + - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod/cache - rm -f modcache_tools.tar.xz .retrieve_linux_go_e2e_deps: - - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod + - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod/cache - rm -f modcache_e2e.tar.xz .cache: @@ -31,6 +31,7 @@ POLICY: pull retry: 2 +# HACK: If you change the behavior of this job, change the `cache:key:prefix` value to invalidate the cache go_deps: extends: .cache variables: @@ -41,26 +42,39 @@ go_deps: # but still provide the artifact that's expected for the other jobs to run - if [ -f modcache.tar.xz ]; then exit 0; fi - inv -e deps --verbose - - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache.tar.xz . + - inv -e install-tools + - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache.tar.xz . artifacts: expire_in: 1 day paths: - $CI_PROJECT_DIR/modcache.tar.xz cache: + # The `cache:key:files` only accepts up to two paths ([docs](https://docs.gitlab.com/ee/ci/yaml/#cachekeyfiles)). + # Ideally, we should also include the https://github.com/DataDog/datadog-agent/blob/main/.custom-gcl.yml file to + # avoid issues if a plugin is added in one PR and enabled in another. However, we decided to accept this limitation + # because the probability for this to happen is very low and go mod files are modified frequently so the risk of + # failing a job because of a network issue when building the custom binary is very low, but still exists. + # We should also include the file this job is defined in to invalicate the cache when this job is modified. - key: files: - go.mod - ./**/go.mod - prefix: "go_deps" + prefix: "go_deps_modcache" paths: - modcache.tar.xz + timeout: 35m go_tools_deps: extends: .cache script: - if [ -f modcache_tools.tar.xz ]; then exit 0; fi - - inv -e download-tools - - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_tools.tar.xz . + - inv -e install-tools + # Partial mitigation for #incident-30779. It will cache datadog-packages , but if the upstream version is updated without the key cache changing, the cache will not work until the key cache changes. + # Long term solution will be to provide the datadog-packages as a binary hosted internally + - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" + - go env -w GOPRIVATE="github.com/DataDog/*" + - go install github.com/DataDog/datadog-packages/cmd/datadog-package@latest + - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_tools.tar.xz . artifacts: expire_in: 1 day paths: @@ -69,7 +83,8 @@ go_tools_deps: - key: files: - ./**/go.mod - prefix: "go_tools_deps" + - .gitlab/deps_fetch/deps_fetch.yml + prefix: "go_tools_deps_modcache" paths: - modcache_tools.tar.xz @@ -78,7 +93,7 @@ go_e2e_deps: script: - if [ -f modcache_e2e.tar.xz ]; then exit 0; fi - inv -e new-e2e-tests.deps - - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz . + - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz . artifacts: expire_in: 1 day paths: @@ -87,6 +102,7 @@ go_e2e_deps: - key: files: - ./test/new-e2e/go.mod - prefix: "go_e2e_deps" + - .gitlab/deps_fetch/deps_fetch.yml + prefix: "go_e2e_deps_modcache" paths: - modcache_e2e.tar.xz diff --git a/.gitlab/dev_container_deploy/docker_linux.yml b/.gitlab/dev_container_deploy/docker_linux.yml index 8d21a4451bc4e..57178929aff04 100644 --- a/.gitlab/dev_container_deploy/docker_linux.yml +++ b/.gitlab/dev_container_deploy/docker_linux.yml @@ -2,25 +2,6 @@ include: - .gitlab/common/container_publish_job_templates.yml -dev_branch-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: !reference [.manual] - needs: - - docker_build_agent6 - - docker_build_agent6_jmx - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG},agent-dev:${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-jmx,agent-dev:${CI_COMMIT_REF_SLUG}-py2-jmx - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-py2py3-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2py3-jmx - dev_branch-dogstatsd: extends: .docker_publish_job_definition stage: dev_container_deploy @@ -33,27 +14,6 @@ dev_branch-dogstatsd: IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 IMG_DESTINATIONS: dogstatsd-dev:${CI_COMMIT_REF_SLUG} -dev_branch_multiarch-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: !reference [.on_all_builds_manual] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG},agent-dev:${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-jmx,agent-dev:${CI_COMMIT_REF_SLUG}-py2-jmx - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-py2py3-jmx-amd64 - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2py3-jmx - dev_branch_multiarch-a7: extends: .docker_publish_job_definition stage: dev_container_deploy @@ -84,25 +44,6 @@ dev_branch_multiarch-dogstatsd: IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 IMG_DESTINATIONS: dogstatsd-dev:${CI_COMMIT_REF_SLUG} -dev_master-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: !reference [.on_main] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:master,agent-dev:master-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:master-jmx,agent-dev:master-py2-jmx - dev_master-a7: extends: .docker_publish_job_definition stage: dev_container_deploy @@ -179,26 +120,6 @@ cws_instrumentation_dev_branch_multiarch: IMG_SOURCES: ${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_CWS_INSTRUMENTATION}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64 IMG_DESTINATIONS: cws-instrumentation-dev:${CI_COMMIT_REF_SLUG} -# deploys nightlies to agent-dev -dev_nightly_docker_hub-a6: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: !reference [.on_deploy_nightly_repo_branch] - needs: - - docker_build_agent6 - - docker_build_agent6_arm64 - - docker_build_agent6_jmx - - docker_build_agent6_jmx_arm64 - - docker_build_agent6_py2py3_jmx - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA},agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py2,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py2 - - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6-jmx-arm64 - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py2-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py2-jmx - # deploys nightlies to agent-dev dev_nightly-a7: extends: .docker_publish_job_definition diff --git a/.gitlab/dev_container_deploy/docker_windows.yml b/.gitlab/dev_container_deploy/docker_windows.yml index 3434cd8036588..850df67aee5b5 100644 --- a/.gitlab/dev_container_deploy/docker_windows.yml +++ b/.gitlab/dev_container_deploy/docker_windows.yml @@ -33,22 +33,6 @@ dev_branch-a7-windows: IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx-win-servercore -dev_branch-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.manual] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py2-win-servercore - dev_master-a7-windows: extends: .docker_publish_job_definition stage: dev_container_deploy @@ -80,22 +64,6 @@ dev_master-a7-windows: IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" IMG_DESTINATIONS: agent-dev:master-py3-jmx-win-servercore -dev_master-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_main] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:master-py2-win-servercore - dev_nightly-a7-windows: extends: .docker_publish_job_definition stage: dev_container_deploy @@ -126,19 +94,3 @@ dev_nightly-a7-windows: - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx" IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py3-jmx-win-servercore - -dev_nightly-a6-windows: - extends: .docker_publish_job_definition - stage: dev_container_deploy - rules: - !reference [.on_deploy_nightly_repo_branch] - needs: - - docker_build_agent6_windows1809_core - - docker_build_agent6_windows2022_core - variables: - IMG_REGISTRIES: dev - parallel: - matrix: - - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-6" - IMG_SOURCES: "%BASE%-win1809-servercore-amd64,%BASE%-winltsc2022-servercore-amd64" - IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_SHORT_SHA}-py2-win-servercore diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index 985274966430d..f760a056b2739 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -139,7 +139,7 @@ new-e2e-agent-subcommands: - EXTRA_PARAMS: --run "Test(Linux|Windows)StatusSuite" - EXTRA_PARAMS: --run "Test(Linux|Windows)HealthSuite" - EXTRA_PARAMS: --run "Test(Linux|Windows)ConfigSuite" - - EXTRA_PARAMS: --run "Test(Linux|Windows)HostnameSuite" + - EXTRA_PARAMS: --run "Test(Linux|Windows|LinuxAzure|LinuxGCP)HostnameSuite" - EXTRA_PARAMS: --run "Test(Linux|Windows)DiagnoseSuite" - EXTRA_PARAMS: --run "Test(Linux|Windows)ConfigCheckSuite" - EXTRA_PARAMS: --run "Test(Linux|Windows)FlareSuite" @@ -288,6 +288,7 @@ new-e2e-orchestrator: variables: TARGETS: ./tests/orchestrator TEAM: container-app + timeout: 55m new-e2e-apm: extends: .new_e2e_template @@ -324,7 +325,7 @@ new-e2e-installer: - qa_installer_oci - qa_agent_oci variables: - TARGETS: ./tests/installer + TARGETS: ./tests/installer/unix TEAM: fleet FLEET_INSTALL_METHOD: "install_script" @@ -343,12 +344,12 @@ new-e2e-installer-windows: # Must run before new_e2e_template changes the aws profile # Note: this is similar to the WINDOWS_AGENT_VERSION in new-e2e_windows_msi but this job is running cross platforms # Note 2: new_e2e_template does not define AGENT_MAJOR_VERSION, so define it as 7 below. - - export CURRENT_AGENT_VERSION=$(invoke agent.version --major-version 7) + - CURRENT_AGENT_VERSION=$(invoke agent.version --major-version 7) || exit $?; export CURRENT_AGENT_VERSION - export STABLE_AGENT_VERSION_PACKAGE=$(curl -sS https://hub.docker.com/v2/namespaces/datadog/repositories/agent-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1) - export STABLE_INSTALLER_VERSION_PACKAGE=$(curl -sS https://hub.docker.com/v2/namespaces/datadog/repositories/installer-package/tags | jq -r '.results[] | .name' | sort | tail -n 2 | head -n 1) - !reference [.new_e2e_template, before_script] variables: - TARGETS: ./tests/installer + TARGETS: ./tests/installer/windows TEAM: fleet FLEET_INSTALL_METHOD: "windows" @@ -368,7 +369,7 @@ new-e2e-installer-ansible: - qa_installer_oci - qa_agent_oci variables: - TARGETS: ./tests/installer + TARGETS: ./tests/installer/unix TEAM: fleet FLEET_INSTALL_METHOD: "ansible" @@ -442,13 +443,6 @@ new-e2e-otel: TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_BRANCH_OSVERS -new-e2e-package-signing-amazonlinux-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_amazonlinux_a7_x86_64 - - .new-e2e_package_signing - rules: !reference [.on_default_new_e2e_tests] - new-e2e-package-signing-debian-a7-x86_64: extends: - .new_e2e_template @@ -463,10 +457,27 @@ new-e2e-package-signing-suse-a7-x86_64: - .new-e2e_package_signing rules: !reference [.on_default_new_e2e_tests] +new-e2e-cspm: + extends: .new_e2e_template + rules: + - !reference [.on_cspm_or_e2e_changes] + - !reference [.manual] + needs: + - !reference [.needs_new_e2e_template] + - qa_agent + - qa_dca + variables: + TARGETS: ./tests/cspm + TEAM: cspm + timeout: 35m + generate-flakes-finder-pipeline: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES stage: e2e - rules: !reference [.on_deploy_nightly_repo_branch] + rules: + - !reference [.except_disable_e2e_tests] + - !reference [.on_deploy_nightly_repo_branch] + - !reference [.manual] needs: - deploy_deb_testing-a7_arm64 - deploy_deb_testing-a7_x64 @@ -481,6 +492,7 @@ generate-flakes-finder-pipeline: - qa_dca - qa_dogstatsd - qa_agent + - qa_agent_ot tags: ["arch:amd64"] script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN @@ -492,7 +504,10 @@ generate-flakes-finder-pipeline: trigger-flakes-finder: stage: e2e needs: [generate-flakes-finder-pipeline] - rules: !reference [.on_deploy_nightly_repo_branch] + rules: + - !reference [.except_disable_e2e_tests] + - !reference [.on_deploy_nightly_repo_branch] + - !reference [.manual] variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID PARENT_COMMIT_SHA: $CI_COMMIT_SHORT_SHA diff --git a/.gitlab/e2e_install_packages/amazonlinux.yml b/.gitlab/e2e_install_packages/amazonlinux.yml index 8cad7d0bc88c3..cb03571ec2ded 100644 --- a/.gitlab/e2e_install_packages/amazonlinux.yml +++ b/.gitlab/e2e_install_packages/amazonlinux.yml @@ -2,26 +2,6 @@ variables: E2E_PLATFORM: amazonlinux -.new-e2e_amazonlinux_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - E2E_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - E2E_BRANCH_OSVERS: "amazonlinux2023" - needs: - - !reference [.needs_new_e2e_template] - - deploy_rpm_testing-a6_x64 - -.new-e2e_amazonlinux_a6_arm64: - variables: - E2E_ARCH: arm64 - E2E_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - E2E_CWS_SUPPORTED_OSVERS: "amazonlinux2-5-10,amazonlinux2022-5-15,amazonlinux2023" - E2E_BRANCH_OSVERS: "amazonlinux2023" - needs: - - !reference [.needs_new_e2e_template] - - deploy_rpm_testing-a6_arm64 - .new-e2e_amazonlinux_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -42,26 +22,6 @@ - !reference [.needs_new_e2e_template] - deploy_rpm_testing-a7_arm64 -new-e2e-agent-platform-install-script-amazonlinux-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_amazonlinux - - .new-e2e_amazonlinux_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-install-script-amazonlinux-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_amazonlinux - - .new-e2e_amazonlinux_a6_arm64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-amazonlinux-a7-x64: extends: - .new_e2e_template @@ -84,30 +44,6 @@ new-e2e-agent-platform-install-script-amazonlinux-a7-arm64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-step-by-step-amazonlinux-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_amazonlinux - - .new-e2e_amazonlinux_a6_x86_64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-step-by-step-amazonlinux-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_amazonlinux - - .new-e2e_amazonlinux_a6_arm64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-step-by-step-amazonlinux-a7-x64: extends: - .new_e2e_template @@ -142,13 +78,13 @@ new-e2e-agent-platform-install-script-upgrade7-amazonlinux-x64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-upgrade6-amazonlinux-x64: +new-e2e-platform-integrations-upgrade7-persisting-integrations-amazonlinux-x64: extends: - .new_e2e_template - - .new-e2e_script_upgrade6 + - .new-e2e_script_upgrade_persisting_integrations - .new-e2e_os_amazonlinux - - .new-e2e_amazonlinux_a6_x86_64 - - .new-e2e_agent_a6 + - .new-e2e_amazonlinux_a7_x86_64 + - .new-e2e_agent_a7 variables: FLAVOR: datadog-agent diff --git a/.gitlab/e2e_install_packages/centos.yml b/.gitlab/e2e_install_packages/centos.yml index cf056465759fb..edb4424eb6fb9 100644 --- a/.gitlab/e2e_install_packages/centos.yml +++ b/.gitlab/e2e_install_packages/centos.yml @@ -2,16 +2,6 @@ variables: E2E_PLATFORM: centos -.new-e2e_centos_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "centos-79,rhel-86" - E2E_CWS_SUPPORTED_OSVERS: "centos-79,rhel-86" - E2E_BRANCH_OSVERS: "centos-79" - needs: - - !reference [.needs_new_e2e_template] - - deploy_rpm_testing-a6_x64 - .new-e2e_centos_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -32,16 +22,6 @@ - !reference [.needs_new_e2e_template] - deploy_rpm_testing-a7_x64 -.new-e2e_centos-fips_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "rhel-86-fips" - E2E_CWS_SUPPORTED_OSVERS: "rhel-86-fips" - E2E_BRANCH_OSVERS: "rhel-86-fips" - needs: - - !reference [.needs_new_e2e_template] - - deploy_rpm_testing-a6_x64 - .new-e2e_centos6_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -52,16 +32,6 @@ - !reference [.needs_new_e2e_template] - deploy_rpm_testing-a7_x64 -new-e2e-agent-platform-install-script-centos-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_centos - - .new-e2e_centos_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-centos-a7-x86_64: extends: - .new_e2e_template @@ -94,16 +64,6 @@ new-e2e-agent-platform-install-script-centos-dogstatsd-a7-x86_64: variables: FLAVOR: datadog-dogstatsd -new-e2e-agent-platform-install-script-centos-fips-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_centos - - .new-e2e_centos-fips_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-centos-fips-a7-x86_64: extends: - .new_e2e_template @@ -136,17 +96,6 @@ new-e2e-agent-platform-install-script-centos-fips-dogstatsd-a7-x86_64: variables: FLAVOR: datadog-dogstatsd -new-e2e-agent-platform-step-by-step-centos-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_centos - - .new-e2e_centos_a6_x86_64 - - .new-e2e_agent_a6 - rules: !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-step-by-step-centos-a7-x86_64: extends: - .new_e2e_template @@ -158,20 +107,20 @@ new-e2e-agent-platform-step-by-step-centos-a7-x86_64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-upgrade6-centos-x86_64: +new-e2e-agent-platform-install-script-upgrade7-centos-x86_64: extends: - .new_e2e_template - - .new-e2e_script_upgrade6 + - .new-e2e_script_upgrade7 - .new-e2e_os_centos - - .new-e2e_centos_a6_x86_64 - - .new-e2e_agent_a6 + - .new-e2e_centos_a7_x86_64 + - .new-e2e_agent_a7 variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-upgrade7-centos-x86_64: +new-e2e-platform-integrations-upgrade7-persisting-integrations-centos-x86_64: extends: - .new_e2e_template - - .new-e2e_script_upgrade7 + - .new-e2e_script_upgrade_persisting_integrations - .new-e2e_os_centos - .new-e2e_centos_a7_x86_64 - .new-e2e_agent_a7 @@ -206,20 +155,6 @@ new-e2e-agent-platform-install-script-upgrade7-centos-fips-iot-agent-x86_64: - START_MAJOR_VERSION: [7] END_MAJOR_VERSION: [7] -new-e2e-agent-platform-install-script-upgrade6-centos-fips-x86_64: - extends: - - .new_e2e_template - - .new-e2e_script_upgrade6 - - .new-e2e_os_centos - - .new-e2e_centos-fips_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - parallel: - matrix: - - START_MAJOR_VERSION: [6] - END_MAJOR_VERSION: [6] - new-e2e-agent-platform-install-script-upgrade7-centos-fips-x86_64: extends: - .new_e2e_template diff --git a/.gitlab/e2e_install_packages/common.yml b/.gitlab/e2e_install_packages/common.yml index a7e887d283f28..1bb6c813d84cc 100644 --- a/.gitlab/e2e_install_packages/common.yml +++ b/.gitlab/e2e_install_packages/common.yml @@ -1,8 +1,3 @@ -.new-e2e_agent_a6: - rules: !reference [.on_kitchen_tests] #TODO: Change when migration is complete to another name without 'kitchen' - variables: - AGENT_MAJOR_VERSION: 6 - .new-e2e_agent_a7: rules: !reference [.on_kitchen_tests] #TODO: Change when migration is complete to another name without 'kitchen' variables: @@ -22,7 +17,7 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --major-version $AGENT_MAJOR_VERSION --arch $E2E_ARCH --flavor $FLAVOR -.new-e2e_script_upgrade6: +.new-e2e_script_upgrade7: stage: e2e_install_packages variables: TARGETS: ./tests/agent-platform/upgrade @@ -30,25 +25,21 @@ EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH --flavor $FLAVOR parallel: matrix: - - START_MAJOR_VERSION: [5, 6] - END_MAJOR_VERSION: [6] + - START_MAJOR_VERSION: [5, 6, 7] + END_MAJOR_VERSION: [7] script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY ) || exit $?; export DATADOG_AGENT_API_KEY + - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION --test-washer -.new-e2e_script_upgrade7: +.new-e2e_script_upgrade_persisting_integrations: stage: e2e_install_packages variables: - TARGETS: ./tests/agent-platform/upgrade + TARGETS: ./tests/agent-platform/persisting-integrations TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH --flavor $FLAVOR - parallel: - matrix: - - START_MAJOR_VERSION: [5, 6, 7] - END_MAJOR_VERSION: [7] script: - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY ) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION + - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version 7 --test-washer .new-e2e_rpm: stage: e2e_install_packages @@ -58,4 +49,4 @@ EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH script: - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY) || exit $?; export DATADOG_AGENT_API_KEY - - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} + - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer diff --git a/.gitlab/e2e_install_packages/debian.yml b/.gitlab/e2e_install_packages/debian.yml index 6a144f733aa25..a07cb0dc8ff56 100644 --- a/.gitlab/e2e_install_packages/debian.yml +++ b/.gitlab/e2e_install_packages/debian.yml @@ -2,26 +2,6 @@ variables: E2E_PLATFORM: debian -.new-e2e_debian_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "debian-9,debian-10,debian-11,debian-12" - E2E_CWS_SUPPORTED_OSVERS: "debian-10,debian-11" - E2E_BRANCH_OSVERS: "debian-11" - needs: - - !reference [.needs_new_e2e_template] - - deploy_deb_testing-a6_x64 - -.new-e2e_debian_a6_arm64: - variables: - E2E_ARCH: arm64 - E2E_OSVERS: "debian-10" - E2E_CWS_SUPPORTED_OSVERS: "debian-10" - E2E_BRANCH_OSVERS: "debian-10" - needs: - - !reference [.needs_new_e2e_template] - - deploy_deb_testing-a6_arm64 - .new-e2e_debian_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -42,26 +22,6 @@ - !reference [.needs_new_e2e_template] - deploy_deb_testing-a7_arm64 -new-e2e-agent-platform-install-script-debian-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_debian - - .new-e2e_debian_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-install-script-debian-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_debian - - .new-e2e_debian_a6_arm64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-debian-a7-x86_64: extends: - .new_e2e_template @@ -84,36 +44,36 @@ new-e2e-agent-platform-install-script-debian-a7-arm64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-debian-iot-agent-a7-x86_64: +new-e2e-platform-integrations-upgrade7-persisting-integrations-debian-x86_64: extends: - .new_e2e_template - - .new-e2e_install_script + - .new-e2e_script_upgrade_persisting_integrations - .new-e2e_os_debian - .new-e2e_debian_a7_x86_64 - .new-e2e_agent_a7 - rules: !reference [.on_default_new_e2e_tests] variables: - FLAVOR: datadog-iot-agent + FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64: +new-e2e-agent-platform-install-script-debian-iot-agent-a7-x86_64: extends: - .new_e2e_template - .new-e2e_install_script - .new-e2e_os_debian - .new-e2e_debian_a7_x86_64 - .new-e2e_agent_a7 + rules: !reference [.on_default_new_e2e_tests] variables: - FLAVOR: datadog-dogstatsd + FLAVOR: datadog-iot-agent -new-e2e-agent-platform-install-script-debian-heroku-agent-a6-x86_64: +new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64: extends: - .new_e2e_template - .new-e2e_install_script - .new-e2e_os_debian - - .new-e2e_debian_a6_x86_64 - - .new-e2e_agent_a6 + - .new-e2e_debian_a7_x86_64 + - .new-e2e_agent_a7 variables: - FLAVOR: datadog-heroku-agent + FLAVOR: datadog-dogstatsd new-e2e-agent-platform-install-script-debian-heroku-agent-a7-x86_64: extends: @@ -149,40 +109,6 @@ new-e2e-agent-platform-step-by-step-debian-a7-arm64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-step-by-step-debian-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_debian - - .new-e2e_debian_a6_x86_64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-step-by-step-debian-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_debian - - .new-e2e_debian_a6_arm64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-install-script-upgrade6-debian-x86_64: - extends: - - .new_e2e_template - - .new-e2e_script_upgrade6 - - .new-e2e_os_debian - - .new-e2e_debian_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-upgrade7-debian-x86_64: extends: - .new_e2e_template diff --git a/.gitlab/e2e_install_packages/suse.yml b/.gitlab/e2e_install_packages/suse.yml index 5187a29e6dcea..ced5856448a5b 100644 --- a/.gitlab/e2e_install_packages/suse.yml +++ b/.gitlab/e2e_install_packages/suse.yml @@ -8,16 +8,6 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --major-version $AGENT_MAJOR_VERSION --arch $E2E_ARCH --flavor $FLAVOR -.new-e2e_suse_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "sles-12,sles-15" - E2E_CWS_SUPPORTED_OSVERS: "sles-12,sles-15" - E2E_BRANCH_OSVERS: "sles-15" - needs: - - !reference [.needs_new_e2e_template] - - deploy_suse_rpm_testing_x64-a6 - .new-e2e_suse_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -38,16 +28,6 @@ - !reference [.needs_new_e2e_template] - deploy_suse_rpm_testing_arm64-a7 -new-e2e-agent-platform-install-script-suse-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_suse - - .new-e2e_suse_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-suse-a7-x86_64: extends: - .new_e2e_template @@ -91,18 +71,6 @@ new-e2e-agent-platform-install-script-suse-dogstatsd-a7-x86_64: variables: FLAVOR: datadog-dogstatsd -new-e2e-agent-platform-step-by-step-suse-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_suse - - .new-e2e_suse_a6_x86_64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-step-by-step-suse-a7-x86_64: extends: - .new_e2e_template @@ -141,20 +109,6 @@ new-e2e-agent-platform-install-script-upgrade7-suse-x86_64: - START_MAJOR_VERSION: [6,7] END_MAJOR_VERSION: [7] -new-e2e-agent-platform-install-script-upgrade6-suse-x86_64: - extends: - - .new_e2e_template - - .new-e2e_script_upgrade7 - - .new-e2e_os_suse - - .new-e2e_suse_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - parallel: - matrix: - - START_MAJOR_VERSION: [6] - END_MAJOR_VERSION: [6] - new-e2e-agent-platform-install-script-upgrade7-suse-iot-agent-x86_64: extends: - .new_e2e_template diff --git a/.gitlab/e2e_install_packages/ubuntu.yml b/.gitlab/e2e_install_packages/ubuntu.yml index a27584b8f1a8c..a7678a9f6ee69 100644 --- a/.gitlab/e2e_install_packages/ubuntu.yml +++ b/.gitlab/e2e_install_packages/ubuntu.yml @@ -15,26 +15,6 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --major-version $AGENT_MAJOR_VERSION --arch $E2E_ARCH --flavor $FLAVOR -.new-e2e_ubuntu_a6_x86_64: - variables: - E2E_ARCH: x86_64 - E2E_OSVERS: "ubuntu-14-04,ubuntu-16-04,ubuntu-18-04,ubuntu-20-04,ubuntu-22-04,ubuntu-24-04" - E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-22-04" - E2E_BRANCH_OSVERS: "ubuntu-24-04" - needs: - - !reference [.needs_new_e2e_template] - - deploy_deb_testing-a6_x64 - -.new-e2e_ubuntu_a6_arm64: - variables: - E2E_ARCH: arm64 - E2E_OSVERS: "ubuntu-18-04,ubuntu-20-04,ubuntu-24-04" - E2E_CWS_SUPPORTED_OSVERS: "ubuntu-18-04,ubuntu-20-04" - E2E_BRANCH_OSVERS: "ubuntu-20-04" - needs: - - !reference [.needs_new_e2e_template] - - deploy_deb_testing-a6_arm64 - .new-e2e_ubuntu_a7_x86_64: variables: E2E_ARCH: x86_64 @@ -55,26 +35,6 @@ - !reference [.needs_new_e2e_template] - deploy_deb_testing-a7_arm64 -new-e2e-agent-platform-install-script-ubuntu-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-install-script-ubuntu-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_install_script - - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_arm64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-ubuntu-a7-x86_64: extends: - .new_e2e_template @@ -99,17 +59,17 @@ new-e2e-agent-platform-install-script-ubuntu-a7-arm64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-ubuntu-iot-agent-a7-x86_64: +new-e2e-platform-integrations-upgrade7-persisting-integrations-ubuntu-x86_64: extends: - .new_e2e_template - - .new-e2e_install_script + - .new-e2e_script_upgrade_persisting_integrations - .new-e2e_os_ubuntu - .new-e2e_ubuntu_a7_x86_64 - .new-e2e_agent_a7 variables: - FLAVOR: datadog-iot-agent + FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-ubuntu-dogstatsd-a7-x86_64: +new-e2e-agent-platform-install-script-ubuntu-iot-agent-a7-x86_64: extends: - .new_e2e_template - .new-e2e_install_script @@ -117,17 +77,17 @@ new-e2e-agent-platform-install-script-ubuntu-dogstatsd-a7-x86_64: - .new-e2e_ubuntu_a7_x86_64 - .new-e2e_agent_a7 variables: - FLAVOR: datadog-dogstatsd + FLAVOR: datadog-iot-agent -new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a6-x86_64: +new-e2e-agent-platform-install-script-ubuntu-dogstatsd-a7-x86_64: extends: - .new_e2e_template - .new-e2e_install_script - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_x86_64 - - .new-e2e_agent_a6 + - .new-e2e_ubuntu_a7_x86_64 + - .new-e2e_agent_a7 variables: - FLAVOR: datadog-heroku-agent + FLAVOR: datadog-dogstatsd new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a7-x86_64: extends: @@ -139,30 +99,6 @@ new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a7-x86_64: variables: FLAVOR: datadog-heroku-agent -new-e2e-agent-platform-step-by-step-ubuntu-a6-x86_64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_x86_64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - -new-e2e-agent-platform-step-by-step-ubuntu-a6-arm64: - extends: - - .new_e2e_template - - .new-e2e_step_by_step - - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_arm64 - - .new-e2e_agent_a6 - rules: - !reference [.on_deploy] - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-step-by-step-ubuntu-a7-x86_64: extends: - .new_e2e_template @@ -197,16 +133,6 @@ new-e2e-agent-platform-install-script-upgrade7-ubuntu-x86_64: variables: FLAVOR: datadog-agent -new-e2e-agent-platform-install-script-upgrade6-ubuntu-x86_64: - extends: - - .new_e2e_template - - .new-e2e_script_upgrade6 - - .new-e2e_os_ubuntu - - .new-e2e_ubuntu_a6_x86_64 - - .new-e2e_agent_a6 - variables: - FLAVOR: datadog-agent - new-e2e-agent-platform-install-script-upgrade7-ubuntu-iot-agent-x86_64: extends: - .new_e2e_template diff --git a/.gitlab/e2e_install_packages/windows.yml b/.gitlab/e2e_install_packages/windows.yml index 70603614abca2..a5585774c9aa5 100644 --- a/.gitlab/e2e_install_packages/windows.yml +++ b/.gitlab/e2e_install_packages/windows.yml @@ -10,7 +10,7 @@ before_script: # WINDOWS_AGENT_VERSION is used to verify the installed agent version # Must run before new_e2e_template changes the aws profile - - export WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) + - WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) || exit $?; export WINDOWS_AGENT_VERSION - !reference [.new_e2e_template, before_script] script: # LAST_STABLE_VERSION is used for upgrade test @@ -24,38 +24,13 @@ before_script: # WINDOWS_AGENT_VERSION is used to verify the installed agent version # Must run before new_e2e_template changes the aws profile - - export WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) + - WINDOWS_AGENT_VERSION=$(invoke agent.version --major-version $AGENT_MAJOR_VERSION) || exit $?; export WINDOWS_AGENT_VERSION - !reference [.new_e2e_template, before_script] script: # LAST_STABLE_VERSION is used for upgrade test - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::$AGENT_MAJOR_VERSION") - !reference [.new_e2e_template, script] -# TODO: gitlab issue prevents using !reference in parallel.matrix so we must duplicate the lists for now. -# https://gitlab.com/gitlab-org/gitlab/-/issues/439828 -.new-e2e_windows_installer_v6_tests: - parallel: - matrix: - - E2E_MSI_TEST: TestInstall - - E2E_MSI_TEST: TestRepair - - E2E_MSI_TEST: TestUpgrade - - E2E_MSI_TEST: TestUpgradeRollback - - E2E_MSI_TEST: TestUpgradeRollbackWithoutCWS - - E2E_MSI_TEST: TestUpgradeChangeUser - - E2E_MSI_TEST: TestUpgradeFromV5 - - E2E_MSI_TEST: TestAgentUser/user_only - - E2E_MSI_TEST: TestAgentUser/dotslash_user - - E2E_MSI_TEST: TestAgentUser/hostname_user - - E2E_MSI_TEST: TestAgentUser/LocalSystem - - E2E_MSI_TEST: TestAgentUser/SYSTEM - - E2E_MSI_TEST: TestInstallOpts - - E2E_MSI_TEST: TestSubServicesOpts/all-subservices - - E2E_MSI_TEST: TestSubServicesOpts/no-subservices - - E2E_MSI_TEST: TestInstallAltDir - - E2E_MSI_TEST: TestInstallExistingAltDir - - E2E_MSI_TEST: TestInstallAltDirAndCorruptForUninstall - - E2E_MSI_TEST: TestInstallFail - .new-e2e_windows_installer_v7_tests: parallel: matrix: @@ -95,28 +70,6 @@ new-e2e_windows_powershell_module_test: needs: - !reference [.needs_new_e2e_template] -# Agent 6 -.new-e2e_windows_a6_x86_64: - variables: - WINDOWS_AGENT_ARCH: "x86_64" - extends: - - .new-e2e_windows_msi - - .new-e2e_agent_a6 - needs: - - !reference [.needs_new_e2e_template] - - deploy_windows_testing-a6 - -## full tests -new-e2e-windows-agent-msi-windows-server-a6-x86_64: - stage: e2e_install_packages - extends: - - .new-e2e_windows_a6_x86_64 - - .new-e2e_windows_installer_v6_tests - rules: - - !reference [.on_deploy] - - !reference [.on_e2e_or_windows_installer_changes] - - !reference [.manual] - # Agent 7 .new-e2e_windows_a7_x86_64: variables: @@ -155,7 +108,7 @@ new-e2e-windows-agent-domain-tests-a7-x86_64: - !reference [.on_deploy] - !reference [.on_e2e_or_windows_installer_changes] - !reference [.manual] - timeout: 1h15 + timeout: 1h15m ## single test for PRs ## skipped if the full tests are running diff --git a/.gitlab/e2e_k8s/e2e_k8s.yml b/.gitlab/e2e_k8s/e2e_k8s.yml deleted file mode 100644 index 45ffd2171a413..0000000000000 --- a/.gitlab/e2e_k8s/e2e_k8s.yml +++ /dev/null @@ -1,70 +0,0 @@ - ---- -# e2e stage -# Jobs with the k8s_e2e template - -.k8s_e2e_template: - stage: e2e_k8s - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - dependencies: [] - variables: - LANG: C.UTF-8 - before_script: - - DOCKER_REGISTRY_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_LOGIN) || exit $?; export DOCKER_REGISTRY_LOGIN - - DOCKER_REGISTRY_PWD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_PWD) || exit $?; export DOCKER_REGISTRY_PWD - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_DDDEV) || exit $?; export DD_API_KEY - -.k8s-e2e-cws-cspm-init: - - set +x - - export DATADOG_AGENT_SITE=datadoghq.com - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_API_KEY) || exit $?; export DATADOG_AGENT_API_KEY - - DATADOG_AGENT_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_APP_KEY) || exit $?; export DATADOG_AGENT_APP_KEY - - DATADOG_AGENT_RC_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_TESTS_RC_KEY) || exit $?; export DATADOG_AGENT_RC_KEY - -.k8s_e2e_template_needs_dev: - extends: .k8s_e2e_template - needs: - - dev_branch_multiarch-a7 - - dca_dev_branch - -.k8s_e2e_template_dev: - extends: .k8s_e2e_template_needs_dev - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=$ARGO_WORKFLOW - -.k8s_e2e_template_dev_with_cws_cspm_init: - extends: .k8s_e2e_template_needs_dev - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 --dca-image=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG} --argo-workflow=$ARGO_WORKFLOW - -.k8s_e2e_template_needs_main: - extends: .k8s_e2e_template - needs: - - dev_master-a7 - - dca_dev_master - -.k8s_e2e_template_main_with_cws_cspm_init: - extends: .k8s_e2e_template_needs_main - script: - - !reference [.k8s-e2e-cws-cspm-init] - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=$ARGO_WORKFLOW - -.k8s_e2e_template_main: - extends: .k8s_e2e_template_needs_main - script: - - inv -e e2e-tests --agent-image=datadog/agent-dev:master-py3 --dca-image=datadog/cluster-agent-dev:master --argo-workflow=$ARGO_WORKFLOW - -k8s-e2e-cspm-dev: - extends: .k8s_e2e_template_dev_with_cws_cspm_init - rules: !reference [.on_dev_branch_manual] - variables: - ARGO_WORKFLOW: cspm - -k8s-e2e-cspm-main: - extends: .k8s_e2e_template_main_with_cws_cspm_init - rules: !reference [.on_main] - retry: 1 - variables: - ARGO_WORKFLOW: cspm diff --git a/.gitlab/e2e_pre_test/e2e_pre_test.yml b/.gitlab/e2e_pre_test/e2e_pre_test.yml index ff172d56d28d9..35af4a004bebe 100644 --- a/.gitlab/e2e_pre_test/e2e_pre_test.yml +++ b/.gitlab/e2e_pre_test/e2e_pre_test.yml @@ -10,7 +10,7 @@ e2e_pre_test: - job: publish_fakeintake optional: true script: - - inv -e new-e2e-tests.run --targets ./test-infra-definition --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} + - inv -e new-e2e-tests.run --targets ./test-infra-definition --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer after_script: - $CI_PROJECT_DIR/tools/ci/junit_upload.sh variables: diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index e0da62daac673..c288fe0824f42 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -1,5 +1,6 @@ single-machine-performance-regression_detector: stage: functional_test + timeout: 1h10m rules: - !reference [.except_main_or_release_branch] - when: on_success @@ -14,7 +15,9 @@ single-machine-performance-regression_detector: - submission_metadata # for provenance, debugging - ${CI_COMMIT_SHA}-baseline_sha # for provenance, debugging - outputs/report.md # for debugging, also on S3 - - outputs/report.html # for debugging, also on S3 + - outputs/regression_signal.json # for debugging, also on S3 + - outputs/bounds_check_signal.json # for debugging, also on S3 + - outputs/junit.xml # for debugging, also on S3 when: always variables: SMP_VERSION: 0.16.0 @@ -33,7 +36,6 @@ single-machine-performance-regression_detector: # Ensure output files exist for artifact downloads step - mkdir outputs # Also needed for smp job sync step - touch outputs/report.md # Will be emitted by smp job sync - - touch outputs/report.html # Will be emitted by smp job sync # Compute merge base of current commit and `main` - git fetch origin - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch) @@ -56,8 +58,38 @@ single-machine-performance-regression_detector: - chmod +x smp - BASELINE_SHA="${SMP_MERGE_BASE}" - echo "Computing baseline..." + - echo "Checking if commit ${BASELINE_SHA} is recent enough..." + # Compute four days before now as UNIX timestamp in order to test against SMP ECR expiration policy; + # add an hour as a small correction factor to overestimate time needed for SMP to query and pull the + # image so we don't end up with a hard-to-diagnose bug in which the image expires after checking its + # age in CI, but before SMP pulls the image. + - FOUR_DAYS_BEFORE_NOW=$(date --date="-4 days +1 hour" "+%s") + # Compute UNIX timestamp of potential baseline SHA + - BASELINE_COMMIT_TIME=$(git -c log.showSignature=false show --no-patch --format=%ct ${BASELINE_SHA}) + # If baseline SHA is older than expiration policy, exit with an error + - | # Only 1st line of multiline command echoes, which reduces debuggability, so multiline commands are a maintenance tradeoff + if [[ ${BASELINE_COMMIT_TIME} -le ${FOUR_DAYS_BEFORE_NOW} ]] + then + echo "ERROR: Merge-base of this branch is too old for SMP. Please update your branch by merging an up-to-date main branch into your branch or by rebasing it on an up-to-date main branch." + exit 1 + fi + - echo "Commit ${BASELINE_SHA} is recent enough" - echo "Checking if image exists for commit ${BASELINE_SHA}..." - - while [[ ! $(aws ecr describe-images --profile single-machine-performance --registry-id "${SMP_ACCOUNT_ID}" --repository-name "${SMP_AGENT_TEAM_ID}-agent" --image-ids imageTag="${BASELINE_SHA}-7-amd64") ]]; do echo "No image exists for ${BASELINE_SHA} - checking predecessor of ${BASELINE_SHA} next"; BASELINE_SHA=$(git rev-parse ${BASELINE_SHA}^); echo "Checking if image exists for commit ${BASELINE_SHA}..."; done + - | + while [[ ! $(aws ecr describe-images --profile single-machine-performance --registry-id "${SMP_ACCOUNT_ID}" --repository-name "${SMP_AGENT_TEAM_ID}-agent" --image-ids imageTag="${BASELINE_SHA}-7-amd64") ]] + do + echo "No image exists for ${BASELINE_SHA} - checking predecessor of ${BASELINE_SHA} next" + BASELINE_SHA=$(git rev-parse ${BASELINE_SHA}^) + echo "Checking if commit ${BASELINE_SHA} is recent enough..." + BASELINE_COMMIT_TIME=$(git -c log.showSignature=false show --no-patch --format=%ct ${BASELINE_SHA}) + if [[ ${BASELINE_COMMIT_TIME} -le ${FOUR_DAYS_BEFORE_NOW} ]] + then + echo "ERROR: Merge-base of this branch is too old for SMP. Please update your branch by merging an up-to-date main branch into your branch or by rebasing it on an up-to-date main branch." + exit 1 + fi + echo "Commit ${BASELINE_SHA} is recent enough" + echo "Checking if image exists for commit ${BASELINE_SHA}..." + done - echo "Image exists for commit ${BASELINE_SHA}" - echo "Baseline SHA is ${BASELINE_SHA}" - echo -n "${BASELINE_SHA}" > "${CI_COMMIT_SHA}-baseline_sha" @@ -92,10 +124,80 @@ single-machine-performance-regression_detector: # space characters. This avoids # https://gitlab.com/gitlab-org/gitlab/-/issues/217231. - cat outputs/report.md | sed "s/^\$/$(echo -ne '\uFEFF\u00A0\u200B')/g" - - !reference [.install_pr_commenter] - # Post HTML report to GitHub - - cat outputs/report.md | /usr/local/bin/pr-commenter --for-pr="$CI_COMMIT_REF_NAME" --header="Regression Detector" + # Upload JUnit XML outside of Agent CI's tooling because the `junit_upload` + # invoke task has additional logic that does not seem to apply well to SMP's + # JUnit XML. Agent CI seems to use `datadog-agent` as the service name when + # uploading JUnit XML, so the upload command below respects that convention. + - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" || exit $?; export DATADOG_API_KEY + - datadog-ci junit upload --service datadog-agent outputs/junit.xml # Finally, exit 1 if the job signals a regression else 0. - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} job result --submission-metadata submission_metadata + +# Shamelessly adapted from golang_deps_commenter job config in +# golang_deps_diff.yml at commit 01da274032e510d617161cf4e264a53292f44e55. +single-machine-performance-regression_detector-pr-comment: + stage: functional_test + rules: + - !reference [.except_main_or_release_branch] + - when: on_success + image: + name: "486234852809.dkr.ecr.us-east-1.amazonaws.com/pr-commenter:3" + entrypoint: [""] # disable entrypoint script for the pr-commenter image + tags: ["arch:amd64"] + needs: + - job: single-machine-performance-regression_detector + artifacts: + expire_in: 1 weeks + paths: + - report_as_json_string.txt # for debugging transform to valid JSON string + - pr_comment_payload.json # for debugging PR commenter JSON payload bugs + variables: + # Not using the entrypoint script for the pr-commenter image + FF_KUBERNETES_HONOR_ENTRYPOINT: false + allow_failure: true # allow_failure here should have same setting as in job above + script: # ignore error message about no PR, because it happens for dev branches without PRs + # We need to transform the Markdown report into a valid JSON string (without + # quotes) in order to pass a well-formed payload to the PR commenting + # service. Note that on macOS, the "-z" flag is invalid for `sed` (but + # should be fine for GNU `sed`). We need to use `sed` to escape newlines + # because JSON does not permit (raw) newlines in strings. We use the "-z" + # option with `sed` because that option treats its input as + # NUL-character-separated (i.e., '\0'-separated, the zero-byte character), + # so `sed` does not interpret its input as newline-delimited. We also need + # to escape double quotes to distinguish literal quotes in the report from + # the double quotes that delimit the value of the "message" field in the + # payload. + - cat outputs/report.md | sed -z 's/\n/\\n/g' | sed -z 's/"/\\"/g' > report_as_json_string.txt + - cat report_as_json_string.txt + # Transforming the Markdown report to a valid JSON string is easy to foul + # up, so to make debugging easier, we store the payload in a variable to + # help debugging. + - PR_COMMENT_JSON_PAYLOAD='{"org":"DataDog", "repo":"datadog-agent", "commit":"'"${CI_COMMIT_SHA}"'", "header":"Regression Detector", "message":"'"$(cat report_as_json_string.txt)"'"}' + - printf "%s\n" "PR comment JSON payload:${PR_COMMENT_JSON_PAYLOAD}" + - printf "%s\n" "${PR_COMMENT_JSON_PAYLOAD}" > pr_comment_payload.json + # Craft an HTTPS request to pr-commenter service to post Markdown report to + # GitHub, per + # https://github.com/DataDog/dd-source/tree/7c941f527fb9c44a73433c7dd0a090d92be7deb4/domains/devex/codex/apps/apis/pr-commenter + # and gracefully handle the case when the commit being tested is not a PR + - | + set +e + out=$(curl https://pr-commenter.us1.ddbuild.io/internal/cit/pr-comment \ + -H "$(authanywhere)" \ + -H "X-DdOrigin: curl" \ + -X PATCH \ + -d "${PR_COMMENT_JSON_PAYLOAD}") + exitcode=$? + set -e + if [ -n "${out}" ]; then + if [ $exitcode -eq 0 ]; then + echo $out + else + echo $out >&2 + fi + fi + if [ "${out}" != "${out/invalid request: no pr found for this commit}" ]; then + exit 0 + fi + exit $exitcode diff --git a/.gitlab/functional_test/security_agent.yml b/.gitlab/functional_test/security_agent.yml index 1842a51081f2d..3d0f71d1fbe60 100644 --- a/.gitlab/functional_test/security_agent.yml +++ b/.gitlab/functional_test/security_agent.yml @@ -101,7 +101,10 @@ kitchen_test_security_agent_amazonlinux_x64: KITCHEN_OSVERS: "amazonlinux2-4-14,amazonlinux2-5-10" KITCHEN_CWS_PLATFORM: [docker] - KITCHEN_PLATFORM: "amazonlinux" - KITCHEN_OSVERS: "amazonlinux2022-5-15,amazonlinux2023" + KITCHEN_OSVERS: "amazonlinux2023" + KITCHEN_CWS_PLATFORM: [docker] + - KITCHEN_PLATFORM: "amazonlinux" + KITCHEN_OSVERS: "amazonlinux2022-5-15" KITCHEN_CWS_PLATFORM: [host, docker] kitchen_test_security_agent_x64_ec2: diff --git a/.gitlab/install_script_testing/install_script_testing.yml b/.gitlab/install_script_testing/install_script_testing.yml index 403625fb08085..8578cba0e42a1 100644 --- a/.gitlab/install_script_testing/install_script_testing.yml +++ b/.gitlab/install_script_testing/install_script_testing.yml @@ -13,8 +13,8 @@ test_install_script: --variable TESTING_APT_URL --variable TESTING_YUM_URL --variable TEST_PIPELINE_ID" - needs: ["deploy_deb_testing-a6_x64", "deploy_rpm_testing-a6_x64", "deploy_suse_rpm_testing_x64-a6", "deploy_deb_testing-a7_x64", "deploy_rpm_testing-a7_x64", "deploy_suse_rpm_testing_x64-a7"] + needs: ["deploy_deb_testing-a7_x64", "deploy_rpm_testing-a7_x64", "deploy_suse_rpm_testing_x64-a7"] rules: - - !reference [.except_no_a6_or_no_a7] + - !reference [.except_no_a7] - !reference [.on_deploy] diff --git a/.gitlab/integration_test/linux.yml b/.gitlab/integration_test/linux.yml new file mode 100644 index 0000000000000..bddbc387e2cd3 --- /dev/null +++ b/.gitlab/integration_test/linux.yml @@ -0,0 +1,25 @@ +.integration_tests_deb: + stage: integration_test + needs: ["go_deps", "go_tools_deps"] + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["docker-in-docker:amd64"] + before_script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + +agent_integration_tests: + extends: .integration_tests_deb + # We have capacity issues on the docker-in-docker:amd64 runners for now. + # Setting allow_failure to true to avoid blocking the pipelines. + # Setting the timeout to 30 min and retry to 2 to avoid the job from pending for too long. + allow_failure: true + timeout: 30m + retry: 2 + script: + - inv -e integration-tests --race --remote-docker + +docker_integration_tests: + extends: .integration_tests_deb + script: + - inv -e docker.test + - inv -e docker.integration-tests diff --git a/.gitlab/integration_test/otel.yml b/.gitlab/integration_test/otel.yml index 511a0cf47cffa..9b372bc8f6766 100644 --- a/.gitlab/integration_test/otel.yml +++ b/.gitlab/integration_test/otel.yml @@ -6,8 +6,9 @@ integration_tests_otel: stage: integration_test image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - needs: [] + needs: ["go_deps"] script: + - !reference [.retrieve_linux_go_deps] - inv check-otel-build - inv check-otel-module-versions rules: @@ -20,3 +21,36 @@ integration_tests_otel: - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/ when: never - when: always + + +docker_image_build_otel: + stage: integration_test + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + needs: ["go_deps","integration_tests_otel"] + tags: ["runner:docker"] + before_script: + - !reference [.retrieve_linux_go_deps] + - mkdir -p /tmp/otel-ci + - cp comp/otelcol/collector-contrib/impl/manifest.yaml /tmp/otel-ci/ + - cp Dockerfiles/agent-ot/Dockerfile.agent-otel /tmp/otel-ci/ + - cp test/integration/docker/otel_agent_build_tests.py /tmp/otel-ci/ + - wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O + /usr/bin/yq && chmod +x /usr/bin/yq + - export OTELCOL_VERSION=v$(/usr/bin/yq r /tmp/otel-ci/manifest.yaml dist.otelcol_version) + - yq w -i /tmp/otel-ci/manifest.yaml "receivers[+] gomod" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sobjectsreceiver ${OTELCOL_VERSION}" + - yq w -i /tmp/otel-ci/manifest.yaml "processors[+] gomod" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor ${OTELCOL_VERSION}" + script: + - docker build -t agent-byoc:latest -f /tmp/otel-ci/Dockerfile.agent-otel /tmp/otel-ci + - OT_AGENT_IMAGE_NAME=agent-byoc OT_AGENT_TAG=latest python3 /tmp/otel-ci/otel_agent_build_tests.py + rules: + - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/ + when: never + - if: $CI_COMMIT_TAG + when: never + - if: $CI_COMMIT_MESSAGE =~ /.*\[skip cancel\].*/ + when: never + - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/ + when: never + - when: always diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml index 127454f00688f..2ddf04fddb997 100644 --- a/.gitlab/integration_test/windows.yml +++ b/.gitlab/integration_test/windows.yml @@ -8,7 +8,7 @@ tags: ["runner:windows-docker", "windowsversion:1809"] before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:VCPKG_BLOB_SAS_URL" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:VCPKG_BLOB_SAS_URL" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $vcpkgBlobSaSUrl=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/.gitlab/internal_image_deploy/internal_image_deploy.yml b/.gitlab/internal_image_deploy/internal_image_deploy.yml index 687c4671defe8..6d0cc3245e3b5 100644 --- a/.gitlab/internal_image_deploy/internal_image_deploy.yml +++ b/.gitlab/internal_image_deploy/internal_image_deploy.yml @@ -14,7 +14,7 @@ docker_trigger_internal: tags: ["arch:amd64"] variables: DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v10 + IMAGE_VERSION: tmpl-v11 IMAGE_NAME: datadog-agent RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-jmx BUILD_TAG: ${CI_COMMIT_REF_SLUG}-jmx @@ -59,7 +59,7 @@ docker_trigger_internal-ot: tags: ["arch:amd64"] variables: DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo - IMAGE_VERSION: tmpl-v10 + IMAGE_VERSION: tmpl-v11 IMAGE_NAME: datadog-agent RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-ot-beta-jmx BUILD_TAG: ${CI_COMMIT_REF_SLUG}-ot-beta-jmx diff --git a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml index 59bd1508d7c6a..605ac0def4114 100644 --- a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml @@ -68,4 +68,4 @@ notify-slack: script: - export SDM_JWT=$(vault read -field=token identity/oidc/token/sdm) - python3 -m pip install -r tasks/requirements.txt - - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} + - inv pipeline.changelog ${CI_COMMIT_SHORT_SHA} || exit $? diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index bbe512a2bf8a9..b2acf5a35d422 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -142,11 +142,13 @@ KUBERNETES_MEMORY_REQUEST: "12Gi" KUBERNETES_MEMORY_LIMIT: "16Gi" VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json" + EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json before_script: - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - !reference [.retrieve_linux_go_deps] - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] + - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH || true script: - echo "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" > $STACK_DIR - pulumi login $(cat $STACK_DIR | tr -d '\n') @@ -189,6 +191,9 @@ - $CI_PROJECT_DIR/stack.output - $CI_PROJECT_DIR/libvirt - $VMCONFIG_FILE + reports: + annotations: + - $EXTERNAL_LINKS_PATH .kmt_cleanup: stage: kernel_matrix_testing_cleanup @@ -329,10 +334,7 @@ notify_ebpf_complexity_changes: before_script: - python3 -m pip install tabulate # Required for printing the tables - python3 -m pip install -r tasks/libs/requirements-github.txt - - | - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_APP_KEY | base64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_INTEGRATION_ID) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - !reference [.setup_agent_github_app] + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN script: - inv -e ebpf.generate-complexity-summary-for-pr diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index c75f78c5449d8..48a2f9de606a9 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -182,7 +182,7 @@ kmt_run_secagent_tests_arm64: - "oracle_9.3" - "rocky_8.5" - "rocky_9.3" - - "opensuse_15.5" + # - "opensuse_15.5" TEST_SET: ["all_tests"] after_script: - !reference [.collect_outcomes_kmt] diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index 7d8da51e0e0ad..da9c47e167702 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -28,8 +28,8 @@ upload_dependencies_sysprobe_arm64: stage: kernel_matrix_testing_prepare script: # DockerHub login for build to limit rate limit when pulling base images - - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_LOGIN) || exit $? - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_PWD | crane auth login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" + - DOCKER_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO user) || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_RO token | crane auth login --username "$DOCKER_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" - EXIT="${PIPESTATUS[0]}"; if [ $EXIT -ne 0 ]; then echo "Unable to locate credentials needs gitlab runner restart"; exit $EXIT; fi # Pull base images - mkdir $KMT_DOCKERS diff --git a/.gitlab/kitchen_cleanup/cleanup.yml b/.gitlab/kitchen_cleanup/cleanup.yml index fbf2d5cde4403..ae0760f29faed 100644 --- a/.gitlab/kitchen_cleanup/cleanup.yml +++ b/.gitlab/kitchen_cleanup/cleanup.yml @@ -8,7 +8,7 @@ - aws s3 rm s3://$DEB_TESTING_S3_BUCKET/dists/pipeline-$DD_PIPELINE_ID --recursive - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/testing/pipeline-$DD_PIPELINE_ID --recursive - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/testing/suse/pipeline-$DD_PIPELINE_ID --recursive - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi + - export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7 - aws s3 rm s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET --recursive - cd $OMNIBUS_PACKAGE_DIR # Remove all deb packages for the pipeline in the pool diff --git a/.gitlab/kitchen_cleanup/kitchen_cleanup.yml b/.gitlab/kitchen_cleanup/kitchen_cleanup.yml index 958c600db26d6..ff3634ad24d94 100644 --- a/.gitlab/kitchen_cleanup/kitchen_cleanup.yml +++ b/.gitlab/kitchen_cleanup/kitchen_cleanup.yml @@ -8,13 +8,6 @@ # include: # - .gitlab/kitchen_common/cleanup.yml -kitchen_cleanup_azure-a6: - extends: .kitchen_cleanup_azure_common - rules: - !reference [.on_kitchen_tests_always] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - kitchen_cleanup_azure-a7: extends: .kitchen_cleanup_azure_common rules: diff --git a/.gitlab/kitchen_deploy/kitchen_deploy.yml b/.gitlab/kitchen_deploy/kitchen_deploy.yml index cc8657b6bbd39..cc35f32c30a69 100644 --- a/.gitlab/kitchen_deploy/kitchen_deploy.yml +++ b/.gitlab/kitchen_deploy/kitchen_deploy.yml @@ -32,50 +32,6 @@ - filename=$(ls datadog-signing-keys*.deb); mv $filename datadog-signing-keys_${DD_PIPELINE_ID}.deb - popd -.deploy_deb_testing-a6: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - ls $OMNIBUS_PACKAGE_DIR - -deploy_deb_testing-a6_x64: - rules: !reference [.on_kitchen_tests] - extends: - - .deploy_deb_testing-a6 - needs: - [ - "agent_deb-x64-a6", - "agent_heroku_deb-x64-a6", - "lint_linux-x64", - ] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*amd64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-x86_64" -m 6 -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - -deploy_deb_testing-a6_arm64: - rules: !reference [.on_all_install_script_tests] - extends: - - .deploy_deb_testing-a6 - needs: ["agent_deb-arm64-a6"] - script: - - *setup_apt_signing_key - - set +x # make sure we don't output the creds to the build log - - - *setup_signing_keys_package - - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 6 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_6*arm64.deb - - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 6 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb - .deploy_deb_testing-a7: stage: kitchen_deploy image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -129,43 +85,6 @@ deploy_deb_testing-a7_arm64: - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-*_7*arm64.deb - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb -.deploy_rpm_testing-a6: - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - ls $OMNIBUS_PACKAGE_DIR - -deploy_rpm_testing-a6_x64: - rules: !reference [.on_kitchen_tests] - extends: - - .deploy_rpm_testing-a6 - needs: - [ - "agent_rpm-x64-a6", - "lint_linux-x64", - ] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | python2 /opt/rpm-s3/bin/rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*x86_64.rpm - -deploy_rpm_testing-a6_arm64: - rules: !reference [.on_all_install_script_tests] - extends: - - .deploy_rpm_testing-a6 - needs: - [ - "agent_rpm-arm64-a6", - "lint_linux-arm64", - ] - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | python2 /opt/rpm-s3/bin/rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/6/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-6.*aarch64.rpm - .deploy_rpm_testing-a7: stage: kitchen_deploy image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -208,25 +127,6 @@ deploy_rpm_testing-a7_arm64: - set +x - echo "$RPM_SIGNING_PASSPHRASE" | python2 /opt/rpm-s3/bin/rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "testing/pipeline-$DD_PIPELINE_ID/7/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID $OMNIBUS_PACKAGE_DIR/datadog-*-7.*aarch64.rpm -deploy_suse_rpm_testing_x64-a6: - rules: !reference [.on_kitchen_tests] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: - [ - "agent_suse-x64-a6", - "lint_linux-x64", - ] - variables: - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - before_script: - - ls $OMNIBUS_PACKAGE_DIR_SUSE - script: - - *setup_rpm_signing_key - - set +x - - echo "$RPM_SIGNING_PASSPHRASE" | python2 /opt/rpm-s3/bin/rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "suse/testing/pipeline-$DD_PIPELINE_ID/6/x86_64/" -a "x86_64" --sign --metadata-signing-key $RPM_GPG_KEY_ID --repodata-store-public-key $OMNIBUS_PACKAGE_DIR_SUSE/datadog-*-6.*x86_64.rpm - deploy_suse_rpm_testing_x64-a7: rules: - !reference [.except_no_tests_no_deploy] @@ -270,20 +170,6 @@ deploy_suse_rpm_testing_arm64-a7: - set +x - echo "$RPM_SIGNING_PASSPHRASE" | python2 /opt/rpm-s3/bin/rpm-s3 --verbose --visibility public-read -c "https://s3.amazonaws.com" -b $RPM_TESTING_S3_BUCKET -p "suse/testing/pipeline-$DD_PIPELINE_ID/7/aarch64/" -a "aarch64" --sign --metadata-signing-key $RPM_GPG_KEY_ID --repodata-store-public-key $OMNIBUS_PACKAGE_DIR_SUSE/datadog-*-7.*aarch64.rpm -deploy_windows_testing-a6: - rules: - - !reference [.on_kitchen_tests] - - !reference [.on_e2e_or_windows_installer_changes] - - !reference [.manual] - stage: kitchen_deploy - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["lint_windows-x64", "windows_msi_x64-a6"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-6.*.msi" $OMNIBUS_PACKAGE_DIR s3://$WIN_S3_BUCKET/$WINDOWS_TESTING_S3_BUCKET_A6 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - deploy_windows_testing-a7: rules: - !reference [.except_no_tests_no_deploy] diff --git a/.gitlab/kitchen_testing/common.yml b/.gitlab/kitchen_testing/common.yml index f51f0e936435c..a3570520f0637 100644 --- a/.gitlab/kitchen_testing/common.yml +++ b/.gitlab/kitchen_testing/common.yml @@ -76,14 +76,6 @@ # Kitchen: agents # --------------- -.kitchen_agent_a6: - extends: .kitchen_common_with_junit - rules: - !reference [.on_kitchen_tests] - variables: - AGENT_MAJOR_VERSION: 6 - DD_PIPELINE_ID: $CI_PIPELINE_ID-a6 - .kitchen_agent_a7: extends: .kitchen_common_with_junit rules: @@ -104,11 +96,6 @@ script: - tasks/run-test-kitchen.sh upgrade5-test $AGENT_MAJOR_VERSION -.kitchen_test_upgrade6: - script: - - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::6") - - tasks/run-test-kitchen.sh upgrade6-test $AGENT_MAJOR_VERSION - .kitchen_test_upgrade7: script: - export LAST_STABLE_VERSION=$(cd ../.. && invoke release.get-release-json-value "last_stable::7") @@ -167,13 +154,6 @@ - .kitchen_azure_location_central_us - .kitchen_ec2_location_us_east_1 -.kitchen_test_upgrade6_agent: - extends: - - .kitchen_test_upgrade6 - - .kitchen_datadog_agent_flavor - - .kitchen_azure_location_south_central_us - - .kitchen_ec2_location_us_east_1 - .kitchen_test_upgrade7_agent: extends: - .kitchen_test_upgrade7 diff --git a/.gitlab/kitchen_testing/windows.yml b/.gitlab/kitchen_testing/windows.yml index 87346f6cebaa6..54a31a5e77b95 100644 --- a/.gitlab/kitchen_testing/windows.yml +++ b/.gitlab/kitchen_testing/windows.yml @@ -16,7 +16,7 @@ KITCHEN_OSVERS: "win2016,win2019,win2019cn,win2022" DEFAULT_KITCHEN_OSVERS: "win2022" before_script: - - if [ $AGENT_MAJOR_VERSION == "7" ]; then export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7; else export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A6; fi + - export WINDOWS_TESTING_S3_BUCKET=$WINDOWS_TESTING_S3_BUCKET_A7 - cd $DD_AGENT_TESTING_DIR - tasks/kitchen_setup.sh # Windows kitchen tests are slower and more fragile (lots of WinRM::WinRMAuthorizationError and/or execution expired errors) diff --git a/.gitlab/lint/include.yml b/.gitlab/lint/include.yml new file mode 100644 index 0000000000000..a133dcfc7539f --- /dev/null +++ b/.gitlab/lint/include.yml @@ -0,0 +1,9 @@ +# lint stage +# Include job that run linters on the Agent code. + +include: + - .gitlab/lint/technical_linters.yml + - .gitlab/lint/linux.yml + - .gitlab/lint/macos.yml + - .gitlab/lint/windows.yml + diff --git a/.gitlab/lint/linux.yml b/.gitlab/lint/linux.yml new file mode 100644 index 0000000000000..9f85e2c6d851a --- /dev/null +++ b/.gitlab/lint/linux.yml @@ -0,0 +1,56 @@ + +.linux_lint: + stage: lint + needs: ["go_deps", "go_tools_deps"] + variables: + FLAVORS: '--flavor base' + KUBERNETES_CPU_REQUEST: 16 + KUBERNETES_MEMORY_REQUEST: 16Gi + KUBERNETES_MEMORY_LIMIT: 16Gi + script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "3" + - inv -e rtloader.install + - inv -e install-tools + - inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST --debug $FLAVORS $EXTRA_OPTS + +.linux_x64: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + +.linux_arm64: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:arm64"] + +lint_linux-x64: + extends: + - .linux_lint + - .linux_x64 + +lint_linux-arm64: + extends: + - .linux_lint + - .linux_arm64 + +lint_flavor_iot_linux-x64: + extends: + - .linux_lint + - .linux_x64 + variables: + FLAVORS: '--flavor iot' + +lint_flavor_dogstatsd_linux-x64: + extends: + - .linux_lint + - .linux_x64 + variables: + FLAVORS: '--flavor dogstatsd' + +lint_flavor_heroku_linux-x64: + extends: + - .linux_lint + - .linux_x64 + variables: + FLAVORS: '--flavor heroku' + diff --git a/.gitlab/lint/macos.yml b/.gitlab/lint/macos.yml new file mode 100644 index 0000000000000..0743c7c7790cc --- /dev/null +++ b/.gitlab/lint/macos.yml @@ -0,0 +1,27 @@ +--- +include: + - .gitlab/common/macos.yml + +.lint_macos_gitlab: + stage: lint + extends: .macos_gitlab + needs: ["go_deps", "go_tools_deps"] + script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + - inv -e linter.go --cpus 12 --debug --timeout 60 + +lint_macos_gitlab_amd64: + extends: .lint_macos_gitlab + tags: ["macos:monterey-amd64", "specific:true"] + rules: + - !reference [.except_mergequeue] + - when: on_success + +lint_macos_gitlab_arm64: + extends: .lint_macos_gitlab + allow_failure: true + rules: + - !reference [.on_main] + - !reference [.manual] + tags: ["macos:monterey-arm64", "specific:true"] diff --git a/.gitlab/lint/technical_linters.yml b/.gitlab/lint/technical_linters.yml new file mode 100644 index 0000000000000..099cd975bb8a9 --- /dev/null +++ b/.gitlab/lint/technical_linters.yml @@ -0,0 +1,64 @@ + +.lint: + stage: lint + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + +lint_licenses: + extends: .lint + script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + - inv -e install-tools + - inv -e lint-licenses + needs: ["go_tools_deps", "go_deps"] + +lint_shell: + extends: .lint + script: + - inv -e install-shellcheck + - shellcheck --version + #Excludes: + #SC2028: echo may not expand escape sequences. Use printf. + #SC2059: Don't use variables in the printf format string. Use printf "..%s.." "$foo". + - shellcheck --severity=info -e SC2059 -e SC2028 --shell=bash ./cmd/**/*.sh ./omnibus/package-scripts/*/* + +lint_filename: + extends: .lint + script: + - inv -e linter.filenames + +lint_copyrights: + extends: .lint + script: + - inv -e linter.copyrights + +lint_codeowners: + extends: .lint + script: + - inv -e github.lint-codeowner + +lint_components: + extends: .lint + script: + - inv -e lint-components lint-fxutil-oneshot-test + + +lint_python: + extends: .lint + needs: [] + script: + - inv -e linter.python + +lint_update_go: + extends: .lint + needs: [] + script: + - inv -e linter.update-go + +validate_modules: + extends: .lint + needs: [] + script: + - inv -e modules.validate + - inv -e modules.validate-used-by-otel diff --git a/.gitlab/lint/windows.yml b/.gitlab/lint/windows.yml new file mode 100644 index 0000000000000..8423fb7094a60 --- /dev/null +++ b/.gitlab/lint/windows.yml @@ -0,0 +1,35 @@ + + +.lint_windows_base: + stage: lint + needs: ["go_deps", "go_tools_deps"] + tags: ["runner:windows-docker", "windowsversion:1809"] + script: + - $ErrorActionPreference = "Stop" + - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' + # Previously this job required only 8Gb of memory but since Go 1.20 it requires more to avoid being OOM killed. + # Each Windows VM has 32Gb of memory and contains 3 runners that can run one job at a time each (so a maximum of 3 simultaneous jobs per VM). + # Windows jobs are using either 8Gb or 16Gb of memory so we can limit memory to 16Gb on this job because even if we decided to limit to 10Gb for instance, + # it would leave 6Gb free but we could not fit another job with these 6Gb remaining. + - > + docker run --rm + -m 16384M + --storage-opt "size=50GB" + -v "$(Get-Location):c:\mnt" + -e AWS_NETWORKING=true + -e CI_PIPELINE_ID=${CI_PIPELINE_ID} + -e CI_PROJECT_NAME=${CI_PROJECT_NAME} + -e PY_RUNTIMES="$PYTHON_RUNTIMES" + -e GOMODCACHE="c:\modcache" + 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\lint.bat + - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + +lint_windows-x64: + extends: .lint_windows_base + rules: + - !reference [.except_mergequeue] + - when: on_success + variables: + PYTHON_RUNTIMES: 3 + ARCH: "x64" + timeout: 1h30m diff --git a/.gitlab/maintenance_jobs/docker.yml b/.gitlab/maintenance_jobs/docker.yml index d9ef46ad19a73..180219cb9eae0 100644 --- a/.gitlab/maintenance_jobs/docker.yml +++ b/.gitlab/maintenance_jobs/docker.yml @@ -2,26 +2,6 @@ include: - .gitlab/common/container_publish_job_templates.yml -# -# Use these steps to revert the latest tags to a previous release -# while maintaining content trust signatures -# - Create a pipeline on main with the RELEASE_6 and/or RELEASE_7 env vars -# - in the gitlab pipeline view, trigger the step (in the first column) -# -revert_latest_6: - extends: .docker_publish_job_definition - rules: !reference [.on_main_manual] - stage: maintenance_jobs - variables: - NEW_LATEST_RELEASE_6: "" # tag name of the non-jmx version, for example "6.21.0" - IMG_REGISTRIES: public - parallel: - matrix: - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_6} - IMG_DESTINATIONS: agent:6,agent:latest-py2 - - IMG_SOURCES: datadog/agent:${NEW_LATEST_RELEASE_6}-jmx - IMG_DESTINATIONS: agent:6-jmx,agent:latest-py2-jmx - revert_latest_7: extends: .docker_publish_job_definition rules: !reference [.on_main_manual] @@ -43,29 +23,3 @@ revert_latest_7: IMG_DESTINATIONS: dogstatsd:7,dogstatsd:latest - IMG_SOURCES: datadog/cluster-agent:${NEW_LATEST_RELEASE_7} IMG_DESTINATIONS: cluster-agent:latest - -# -# Use this step to delete a tag of a given image -# We call the Docker Hub API because docker cli doesn't support deleting tags -# - Run a pipeline on main with the IMAGE and TAG env vars -# - in the gitlab pipeline view, trigger the step (in the first column) -delete_docker_tag: - rules: !reference [.on_main_manual] - stage: maintenance_jobs - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker-notary:v2718650-9ce6565-0.6.1-py3 - tags: ["arch:amd64"] - dependencies: [] - variables: - IMAGE: "" # image name, for example "agent" - TAG: "" # tag name, for example "6.9.0" - ORGANIZATION: "datadog" - before_script: - - DOCKER_REGISTRY_LOGIN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_LOGIN) || exit $? - - PASS=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DOCKER_REGISTRY_PWD) || exit $? - - python3 -m pip install -r requirements.txt - - | - export DOCKER_TOKEN=`curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'$DOCKER_REGISTRY_LOGIN'", "password": "'$PASS'"}' https://hub.docker.com/v2/users/login/ | python -c 'import sys, json; print(json.load(sys.stdin)["token"].strip())'` - script: - - if [[ -z "$IMAGE" ]]; then echo "Need an image"; exit 1; fi - - if [[ -z "$TAG" ]]; then echo "Need a tag to delete"; exit 1; fi - - inv -e docker.delete ${ORGANIZATION} ${IMAGE} ${TAG} ${DOCKER_TOKEN} &>/dev/null diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index 31e782eda47cc..ac45f8ca6c4cc 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -59,8 +59,8 @@ send_pipeline_stats: notify_github: stage: notify - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:arm64"] rules: - !reference [.except_mergequeue] - !reference [.except_main_or_release_branch] @@ -78,42 +78,33 @@ notify_github: dependencies: [] allow_failure: true script: - - !reference [.install_pr_commenter] + - !reference [.setup_agent_github_app] + - python3 -m pip install -r tasks/libs/requirements-github.txt - messagefile="$(mktemp)" - echo "Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:" >> "$messagefile" - echo '```sh' >> "$messagefile" - echo "inv create-vm --pipeline-id=$CI_PIPELINE_ID --os-family=ubuntu" >> "$messagefile" - echo '```' >> "$messagefile" - 'echo "Note: This applies to commit **$CI_COMMIT_SHORT_SHA**" >> "$messagefile"' - - cat "$messagefile" | /usr/local/bin/pr-commenter --for-pr="$CI_COMMIT_REF_NAME" --header="Test changes on VM" + - inv -e github.pr-commenter --title "Test changes on VM" --body "$(cat "$messagefile")" --echo - rm "$messagefile" notify_gitlab_ci_changes: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES stage: notify - needs: [] + needs: [compute_gitlab_ci_config] tags: ["arch:amd64"] rules: - - if: $CI_PIPELINE_SOURCE != "push" - when: never + - !reference [.compute_gitlab_ci_config_rules] - changes: paths: - .gitlab-ci.yml - .gitlab/**/*.yml compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 - before_script: - # Get main history - - git fetch origin main - - git checkout main - - git checkout - script: - python3 -m pip install -r tasks/libs/requirements-github.txt - - | - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_APP_KEY | base64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_INTEGRATION_ID) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITHUB_PR_COMMENTER_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN - - inv -e notify.gitlab-ci-diff --pr-comment + - !reference [.setup_agent_github_app] + - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment .failure_summary_job: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -163,3 +154,28 @@ notify_failure_summary_daily: echo 'Sending weekly summary' inv -e notify.failure-summary-send-notifications --weekly-summary fi + +close_failing_tests_stale_issues: + stage: notify + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + rules: + # Daily + - if: $CI_COMMIT_BRANCH != "main" || $CI_PIPELINE_SOURCE != "schedule" + when: never + - !reference [.on_deploy_nightly_repo_branch_always] + needs: [] + tags: ["arch:arm64"] + script: + - weekday="$(date --utc '+%A')" + # Weekly on Friday + - | + if [ "$weekday" != "Friday" ]; then + echo "This script is run weekly on Fridays" + exit + fi + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $APP_KEY_ORG2) || exit $?; export DD_APP_KEY + - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD + - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME + - python3 -m pip install -r requirements.txt -r tasks/requirements_release_tasks.txt # For Atlassian / Jira dependencies + - inv -e notify.close-failing-tests-stale-issues diff --git a/.gitlab/package_build/heroku.yml b/.gitlab/package_build/heroku.yml index c036313447986..0b06b6317fa7e 100644 --- a/.gitlab/package_build/heroku.yml +++ b/.gitlab/package_build/heroku.yml @@ -4,7 +4,7 @@ rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] needs: [ @@ -40,6 +40,9 @@ KUBERNETES_CPU_REQUEST: 16 DD_PKG_ARCH: "x86_64" PACKAGE_ARCH: "amd64" + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' + DD_CMAKE_TOOLCHAIN: '/opt/cmake/x86_64-unknown-linux-gnu.toolchain.cmake' artifacts: expire_in: 2 weeks paths: @@ -47,15 +50,6 @@ cache: - !reference [.cache_omnibus_ruby_deps, cache] -agent_heroku_deb-x64-a6: - extends: .heroku_build_base - variables: - DESTINATION_DEB: "datadog-heroku-agent_6_amd64.deb" - DESTINATION_DBG_DEB: "datadog-heroku-agent-dbg_6_amd64.deb" - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: "2,3" - agent_heroku_deb-x64-a7: extends: .heroku_build_base variables: diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 67b3ad894369d..182b649974878 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -5,7 +5,8 @@ .common_build_oci: script: - echo "About to build for $RELEASE_VERSION" - - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/$(inv agent.version -u)-1 + - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - export INSTALL_DIR=/opt/datadog-packages/datadog-agent/"$AGENT_VERSION" - !reference [.retrieve_linux_go_deps] - !reference [.cache_omnibus_ruby_deps, setup] # remove artifacts from previous pipelines that may come from the cache @@ -40,7 +41,7 @@ datadog-agent-oci-x64-a7: - !reference [.except_mergequeue] - when: on_success stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] needs: [ @@ -54,6 +55,9 @@ datadog-agent-oci-x64-a7: PYTHON_RUNTIMES: "3" PACKAGE_ARCH: amd64 DESTINATION_OCI: "datadog-agent-7-remote-updater-amd64.tar.xz" + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' + DD_CMAKE_TOOLCHAIN: '/opt/cmake/x86_64-unknown-linux-gnu.toolchain.cmake' before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 @@ -63,7 +67,7 @@ datadog-agent-oci-arm64-a7: - !reference [.except_mergequeue] - when: on_success stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] needs: [ @@ -77,9 +81,43 @@ datadog-agent-oci-arm64-a7: PYTHON_RUNTIMES: "3" PACKAGE_ARCH: arm64 DESTINATION_OCI: "datadog-agent-7-remote-updater-arm64.tar.xz" + DD_CC: 'aarch64-unknown-linux-gnu-gcc' + DD_CXX: 'aarch64-unknown-linux-gnu-g++' + DD_CMAKE_TOOLCHAIN: '/opt/cmake/aarch64-unknown-linux-gnu.toolchain.cmake' before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 +# +# The bootstrapper program +# +bootstrapper_build: + rules: + - !reference [.except_mergequeue] + - when: on_success + stage: package_build + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + needs: ["go_mod_tidy_check", "go_deps"] + variables: + AGENT_MAJOR_VERSION: 7 + KUBERNETES_CPU_REQUEST: 8 + KUBERNETES_MEMORY_REQUEST: "16Gi" + KUBERNETES_MEMORY_LIMIT: "16Gi" + RELEASE_VERSION: "$RELEASE_VERSION_7" + script: + - !reference [.retrieve_linux_go_deps] + - echo "About to build for $RELEASE_VERSION" + - GOOS=linux GOARCH=amd64 inv -e installer.build --bootstrapper --rebuild --no-cgo --output-bin=$OMNIBUS_PACKAGE_DIR/bootstrapper-linux-amd64 + - GOOS=linux GOARCH=arm64 inv -e installer.build --bootstrapper --rebuild --no-cgo --output-bin=$OMNIBUS_PACKAGE_DIR/bootstrapper-linux-arm64 + - GOOS=windows GOARCH=amd64 inv -e installer.build --bootstrapper --rebuild --no-cgo --output-bin=$OMNIBUS_PACKAGE_DIR/bootstrapper-windows-amd64.exe + - GOOS=darwin GOARCH=amd64 inv -e installer.build --bootstrapper --rebuild --no-cgo --output-bin=$OMNIBUS_PACKAGE_DIR/bootstrapper-darwin-amd64 + - GOOS=darwin GOARCH=arm64 inv -e installer.build --bootstrapper --rebuild --no-cgo --output-bin=$OMNIBUS_PACKAGE_DIR/bootstrapper-darwin-arm64 + - ls -la $OMNIBUS_PACKAGE_DIR + artifacts: + expire_in: 2 weeks + paths: + - $OMNIBUS_PACKAGE_DIR + # # The installer program # @@ -116,13 +154,15 @@ installer-amd64: - !reference [.except_mergequeue] - when: on_success stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: amd64 DESTINATION_FILE: "datadog-installer_7-amd64.tar.xz" RELEASE_VERSION: "$RELEASE_VERSION_7" + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' installer-arm64: extends: .installer_build_common @@ -130,20 +170,23 @@ installer-arm64: - !reference [.except_mergequeue] - when: on_success stage: package_build - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "go_deps"] variables: PACKAGE_ARCH: arm64 DESTINATION_FILE: "datadog-installer_7-arm64.tar.xz" RELEASE_VERSION: "$RELEASE_VERSION_7" + DD_CC: 'aarch64-unknown-linux-gnu-gcc' + DD_CXX: 'aarch64-unknown-linux-gnu-g++' installer-amd64-oci: extends: installer-amd64 variables: DESTINATION_FILE: "datadog-updater_7-amd64-oci.tar.xz" before_script: - - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 + - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/"$AGENT_VERSION" - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" installer-arm64-oci: @@ -151,7 +194,8 @@ installer-arm64-oci: variables: DESTINATION_FILE: "datadog-updater_7-arm64-oci.tar.xz" before_script: - - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/$(inv agent.version -u)-1 + - AGENT_VERSION="$(inv agent.version -u)-1" || exit $? + - export INSTALL_DIR=/opt/datadog-packages/datadog-installer/"$AGENT_VERSION" - export INSTALL_DIR_PARAM="--install-directory=$INSTALL_DIR" windows-installer-amd64: diff --git a/.gitlab/package_build/linux.yml b/.gitlab/package_build/linux.yml index 04aafb966b0e7..e261e31399f13 100644 --- a/.gitlab/package_build/linux.yml +++ b/.gitlab/package_build/linux.yml @@ -1,25 +1,28 @@ +.agent_build_script: + - echo "About to build for $RELEASE_VERSION" + - !reference [.retrieve_linux_go_deps] + - !reference [.cache_omnibus_ruby_deps, setup] + # remove artifacts from previous pipelines that may come from the cache + - rm -rf $OMNIBUS_PACKAGE_DIR/* + # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. + # Thus, we move the artifacts at the end in a gitlab-friendly dir. + - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz + - mkdir -p /tmp/system-probe + - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf + - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf + - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz + - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf + - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" --config-directory "$CONFIG_DIR" --install-directory "$INSTALL_DIR" + - ls -la $OMNIBUS_PACKAGE_DIR + - !reference [.upload_sbom_artifacts] + .agent_build_common: rules: - !reference [.except_mergequeue] - when: on_success stage: package_build script: - - echo "About to build for $RELEASE_VERSION" - - !reference [.retrieve_linux_go_deps] - - !reference [.cache_omnibus_ruby_deps, setup] - # remove artifacts from previous pipelines that may come from the cache - - rm -rf $OMNIBUS_PACKAGE_DIR/* - # Artifacts and cache must live within project directory but we run omnibus in a neutral directory. - # Thus, we move the artifacts at the end in a gitlab-friendly dir. - - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz - - mkdir -p /tmp/system-probe - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf - - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf - - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz - - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf - - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --python-runtimes "$PYTHON_RUNTIMES" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" - - ls -la $OMNIBUS_PACKAGE_DIR - - !reference [.upload_sbom_artifacts] + - !reference [.agent_build_script] variables: KUBERNETES_CPU_REQUEST: 16 KUBERNETES_MEMORY_REQUEST: "32Gi" @@ -32,7 +35,7 @@ - !reference [.cache_omnibus_ruby_deps, cache] .agent_build_x86: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] needs: [ @@ -43,9 +46,12 @@ ] variables: PACKAGE_ARCH: amd64 + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' + DD_CMAKE_TOOLCHAIN: '/opt/cmake/x86_64-unknown-linux-gnu.toolchain.cmake' .agent_build_arm64: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] needs: [ @@ -56,14 +62,9 @@ ] variables: PACKAGE_ARCH: arm64 - -.agent_6_build: - variables: - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: "2,3" - FLAVOR: base - before_script: - - export RELEASE_VERSION=$RELEASE_VERSION_6 + DD_CC: 'aarch64-unknown-linux-gnu-gcc' + DD_CXX: 'aarch64-unknown-linux-gnu-g++' + DD_CMAKE_TOOLCHAIN: '/opt/cmake/aarch64-unknown-linux-gnu.toolchain.cmake' .agent_7_build: variables: @@ -81,18 +82,40 @@ before_script: - export RELEASE_VERSION=$RELEASE_VERSION_7 -# build Agent 6 binaries for x86_64 -datadog-agent-6-x64: - extends: [.agent_build_common, .agent_build_x86, .agent_6_build] +# Temporary custom agent build test to prevent regression +# This test will be removed when custom path are used to build macos agent +# with in-house macos runner builds. +datadog-agent-7-x64-custom-path-test: + extends: [.agent_build_x86, .agent_7_build] + rules: + - !reference [.except_mergequeue] + - when: on_success + stage: package_build + script: + - mkdir /custom + - export CONFIG_DIR="/custom" + - export INSTALL_DIR="/custom/datadog-agent" + - !reference [.agent_build_script] + - ls -la $OMNIBUS_PACKAGE_DIR + - ls -la $INSTALL_DIR + - ls -la /custom/etc + - (ls -la /opt/datadog-agent 2>/dev/null && exit 1) || echo "/opt/datadog-agent has correctly not been generated" + - (ls -la /etc/datadog-agent 2>/dev/null && exit 1) || echo "/etc/datadog-agent has correctly not been generated" + variables: + KUBERNETES_CPU_REQUEST: 16 + KUBERNETES_MEMORY_REQUEST: "32Gi" + KUBERNETES_MEMORY_LIMIT: "32Gi" + artifacts: + expire_in: 2 weeks + paths: + - $OMNIBUS_PACKAGE_DIR + cache: + - !reference [.cache_omnibus_ruby_deps, cache] # build Agent 7 binaries for x86_64 datadog-agent-7-x64: extends: [.agent_build_common, .agent_build_x86, .agent_7_build] -# build Agent 6 binaries for arm64 -datadog-agent-6-arm64: - extends: [.agent_build_common, .agent_build_arm64, .agent_6_build] - # build Agent 7 binaries for arm64 datadog-agent-7-arm64: extends: [.agent_build_common, .agent_build_arm64, .agent_7_build] @@ -123,12 +146,18 @@ datadog-ot-agent-7-arm64: iot-agent-x64: extends: .iot-agent-common tags: ["arch:amd64"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + variables: + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' iot-agent-arm64: extends: .iot-agent-common tags: ["arch:arm64"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + variables: + DD_CC: 'aarch64-unknown-linux-gnu-gcc' + DD_CXX: 'aarch64-unknown-linux-gnu-g++' iot-agent-armhf: extends: .iot-agent-common @@ -169,12 +198,18 @@ iot-agent-armhf: dogstatsd-x64: extends: .dogstatsd_build_common needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"] - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] + variables: + DD_CC: 'x86_64-unknown-linux-gnu-gcc' + DD_CXX: 'x86_64-unknown-linux-gnu-g++' dogstatsd-arm64: extends: .dogstatsd_build_common - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"] + variables: + DD_CC: 'aarch64-unknown-linux-gnu-gcc' + DD_CXX: 'aarch64-unknown-linux-gnu-g++' diff --git a/.gitlab/package_build/windows.yml b/.gitlab/package_build/windows.yml index b54b3c33eb677..0f265cb260c83 100644 --- a/.gitlab/package_build/windows.yml +++ b/.gitlab/package_build/windows.yml @@ -67,19 +67,6 @@ windows_msi_and_bosh_zip_x64-a7: - set RELEASE_VERSION $RELEASE_VERSION_7 timeout: 2h -windows_msi_x64-a6: - extends: .windows_main_agent_base - rules: - - !reference [.except_mergequeue] - - when: on_success - variables: - ARCH: "x64" - AGENT_MAJOR_VERSION: 6 - PYTHON_RUNTIMES: "2,3" - before_script: - - set RELEASE_VERSION $RELEASE_VERSION_6 - timeout: 2h10 - # cloudfoundry IoT build for Windows windows_zip_agent_binaries_x64-a7: stage: package_build diff --git a/.gitlab/packaging/deb.yml b/.gitlab/packaging/deb.yml index b45aad5855716..1bed3c58a89cc 100644 --- a/.gitlab/packaging/deb.yml +++ b/.gitlab/packaging/deb.yml @@ -35,26 +35,11 @@ PACKAGE_ARCH: arm64 DD_PKG_ARCH: arm64 -.package_deb_agent_6: - variables: - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 - .package_deb_agent_7: variables: RELEASE_VERSION: $RELEASE_VERSION_7 AGENT_MAJOR_VERSION: 7 -agent_deb-x64-a6: - extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_6] - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["datadog-agent-6-x64"] - variables: - DESTINATION_DEB: "datadog-agent_6_amd64.deb" - DD_PROJECT: "agent" - agent_deb-x64-a7: extends: [.package_deb_common, .package_deb_x86, .package_deb_agent_7] rules: @@ -65,16 +50,6 @@ agent_deb-x64-a7: DESTINATION_DEB: "datadog-agent_7_amd64.deb" DD_PROJECT: "agent" -agent_deb-arm64-a6: - extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_6] - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: ["datadog-agent-6-arm64"] - variables: - DESTINATION_DEB: "datadog-agent_6_arm64.deb" - DD_PROJECT: "agent" - agent_deb-arm64-a7: extends: [.package_deb_common, .package_deb_arm64, .package_deb_agent_7] rules: diff --git a/.gitlab/packaging/oci.yml b/.gitlab/packaging/oci.yml index b2dcd8eaf740a..0efb48a3e6e0e 100644 --- a/.gitlab/packaging/oci.yml +++ b/.gitlab/packaging/oci.yml @@ -6,13 +6,14 @@ image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] before_script: - - export PACKAGE_VERSION=$(inv agent.version --url-safe --major-version 7)-1 + - PACKAGE_VERSION="$(inv agent.version --url-safe --major-version 7)-1" || exit $? - export INSTALL_DIR=/opt/datadog-packages/${OCI_PRODUCT}/${PACKAGE_VERSION} variables: KUBERNETES_CPU_REQUEST: 16 KUBERNETES_MEMORY_REQUEST: "32Gi" KUBERNETES_MEMORY_LIMIT: "32Gi" script: + - !reference [.retrieve_linux_go_tools_deps] - rm -f $OMNIBUS_PACKAGE_DIR/*-dbg-*.tar.xz - ls -l $OMNIBUS_PACKAGE_DIR - python3 -m pip install -r tasks/libs/requirements-github.txt @@ -67,12 +68,12 @@ agent_oci: extends: .package_oci - needs: ["datadog-agent-oci-x64-a7", "datadog-agent-oci-arm64-a7", "windows_msi_and_bosh_zip_x64-a7"] + needs: ["datadog-agent-oci-x64-a7", "datadog-agent-oci-arm64-a7", "windows_msi_and_bosh_zip_x64-a7", "go_tools_deps"] variables: OCI_PRODUCT: "datadog-agent" installer_oci: extends: .package_oci - needs: ["installer-arm64-oci", "installer-amd64-oci", "windows-installer-amd64"] + needs: ["installer-arm64-oci", "installer-amd64-oci", "windows-installer-amd64", "go_tools_deps"] variables: OCI_PRODUCT: "datadog-installer" diff --git a/.gitlab/packaging/rpm.yml b/.gitlab/packaging/rpm.yml index 7ab26f7d86e9b..c8617de6dcbf8 100644 --- a/.gitlab/packaging/rpm.yml +++ b/.gitlab/packaging/rpm.yml @@ -42,11 +42,6 @@ PACKAGE_ARCH: arm64 DD_PKG_ARCH: arm64 -.package_rpm_agent_6: - variables: - RELEASE_VERSION: $RELEASE_VERSION_6 - AGENT_MAJOR_VERSION: 6 - .package_rpm_agent_7: variables: RELEASE_VERSION: $RELEASE_VERSION_7 @@ -64,12 +59,6 @@ variables: OMNIBUS_EXTRA_ARGS: "--host-distribution=suse" -agent_rpm-x64-a6: - extends: [.package_rpm_common, .package_rpm_agent_6, .package_rpm_x86] - needs: ["datadog-agent-6-x64"] - variables: - DD_PROJECT: agent - agent_rpm-x64-a7: extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_x86] tags: ["arch:amd64"] @@ -77,24 +66,12 @@ agent_rpm-x64-a7: variables: DD_PROJECT: agent -agent_rpm-arm64-a6: - extends: [.package_rpm_common, .package_rpm_agent_6, .package_rpm_arm64] - needs: ["datadog-agent-6-arm64"] - variables: - DD_PROJECT: agent - agent_rpm-arm64-a7: extends: [.package_rpm_common, .package_rpm_agent_7, .package_rpm_arm64] needs: ["datadog-agent-7-arm64"] variables: DD_PROJECT: agent -agent_suse-x64-a6: - extends: [.package_suse_rpm_common, .package_rpm_agent_6, .package_rpm_x86] - needs: ["datadog-agent-6-x64"] - variables: - DD_PRODUCT: agent - agent_suse-x64-a7: extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86] needs: ["datadog-agent-7-x64"] diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index 21c647bbcd805..7001321d669c4 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -11,30 +11,18 @@ send_pkg_size: - when: always allow_failure: true needs: - - job: agent_deb-arm64-a6 - optional: true - job: agent_deb-arm64-a7 optional: true - - job: agent_deb-x64-a6 - optional: true - job: agent_deb-x64-a7 optional: true - - job: agent_heroku_deb-x64-a6 - optional: true - job: agent_heroku_deb-x64-a7 optional: true - - job: agent_rpm-arm64-a6 - optional: true - job: agent_rpm-arm64-a7 optional: true - - job: agent_rpm-x64-a6 - optional: true - job: agent_rpm-x64-a7 optional: true - job: agent_suse-arm64-a7 optional: true - - job: agent_suse-x64-a6 - optional: true - job: agent_suse-x64-a7 optional: true - job: dogstatsd_deb-arm64 @@ -65,18 +53,6 @@ send_pkg_size: - ls -l $OMNIBUS_PACKAGE_DIR - ls -l $OMNIBUS_PACKAGE_DIR_SUSE - # Agent 6 - - inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_6*_amd64.deb --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - inv package.send-size --flavor "heroku-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent_6*_amd64.deb --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - - inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_6*_arm64.deb --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - - inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-6.*.x86_64.rpm --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - - - inv package.send-size --flavor "agent" --package-os "centos" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent-6.*.aarch64.rpm --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 - - - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-6.*.x86_64.rpm --major-version "6" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - # Agent 7 - inv package.send-size --flavor "agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 - inv package.send-size --flavor "iot-agent" --package-os "debian" --package-path $OMNIBUS_PACKAGE_DIR/datadog-iot-agent_7*_amd64.deb --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch amd64 @@ -149,45 +125,6 @@ send_pkg_size: # Make the job fail if at least one package is above threshold - if [ "${failures}" -ne "0" ]; then false; fi -check_pkg_size-amd64-a6: - extends: .check_pkg_size - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - agent_deb-x64-a6 - - agent_rpm-x64-a6 - - agent_suse-x64-a6 - variables: - MAJOR_VERSION: 6 - FLAVORS: "datadog-agent" - ARCH: "amd64" - before_script: - # FIXME: ["datadog-agent"]="140000000" should be replaced by "50000000" - # "140000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ) - -check_pkg_size-arm64-a6: - extends: .check_pkg_size - rules: !reference [.on_all_builds] - needs: - - agent_deb-arm64-a6 - - agent_rpm-arm64-a6 - variables: - MAJOR_VERSION: 6 - FLAVORS: "datadog-agent" - ARCH: "arm64" - before_script: - # FIXME: ["datadog-agent"]="140000000" should be replaced by "50000000" - # "140000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ) - check_pkg_size-amd64-a7: extends: .check_pkg_size rules: diff --git a/.gitlab/post_rc_build/post_rc_tasks.yml b/.gitlab/post_rc_build/post_rc_tasks.yml index 3c9ca13b377f4..f02bda3516650 100644 --- a/.gitlab/post_rc_build/post_rc_tasks.yml +++ b/.gitlab/post_rc_build/post_rc_tasks.yml @@ -11,8 +11,8 @@ update_rc_build_links: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: - - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $JIRA_READ_API_TOKEN) || exit $?; export ATLASSIAN_PASSWORD - - ATLASSIAN_USERNAME=robot-jira-agentplatform@datadoghq.com; export ATLASSIAN_USERNAME + - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD + - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - python3 -m pip install -r tasks/requirements_release_tasks.txt - PATCH=$(echo "$CI_COMMIT_REF_NAME" | cut -d'.' -f3 | cut -c1) - if [[ "$PATCH" == "0" ]]; then PATCH_OPTION=""; else PATCH_OPTION="-p"; fi diff --git a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml new file mode 100644 index 0000000000000..53fce34835b75 --- /dev/null +++ b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml @@ -0,0 +1,28 @@ +# We could (should?) piggy back on deploy_installer_packages_windows-x64 to also deploy this +# script to $S3_RELEASE_INSTALLER_ARTIFACTS_URI and have the agent-release-management repository +# publish it to a production bucket like ddagent-windows-stable. +# For now we can use the dd-agent-mstesting bucket to store the PowerShell script. +powershell_script_deploy: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + stage: choco_and_install_script_deploy + rules: + !reference [.manual] + needs: ["powershell_script_signing"] + script: + - ls $WINDOWS_POWERSHELL_DIR + - $S3_CP_CMD $WINDOWS_POWERSHELL_DIR/Install-Datadog.ps1 s3://dd-agent-mstesting/Install-Datadog.ps1 --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 + +# Technically deploy_installer_packages_windows-x64 also uploads the bootstrapper to $S3_RELEASE_INSTALLER_ARTIFACTS_URI +# but it requires changes in agent-release-management to deploy to a production bucket like ddagent-windows-stable. +# For now we can use the dd-agent-mstesting bucket to store the bootstrapper. +windows_bootstrapper_deploy: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] + stage: choco_and_install_script_deploy + rules: + !reference [.manual] + needs: ["windows-installer-amd64"] + script: + - ls $OMNIBUS_PACKAGE_DIR + - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-installer-*-1-x86_64.exe s3://dd-agent-mstesting/datadog-installer-x86_64.exe --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 diff --git a/.gitlab/powershell_script_signing/powershell_script_signing.yml b/.gitlab/powershell_script_signing/powershell_script_signing.yml new file mode 100644 index 0000000000000..c2985179eee4c --- /dev/null +++ b/.gitlab/powershell_script_signing/powershell_script_signing.yml @@ -0,0 +1,16 @@ +powershell_script_signing: + tags: ["runner:windows-docker", "windowsversion:1809"] + stage: choco_and_install_script_build + needs: [] + variables: + ARCH: "x64" + rules: + !reference [.manual] + artifacts: + expire_in: 2 weeks + paths: + - $WINDOWS_POWERSHELL_DIR + script: + - mkdir $WINDOWS_POWERSHELL_DIR + - docker run --rm -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e IS_AWS_CONTAINER=true 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} powershell -C "dd-wcs sign \mnt\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1" + - copy .\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1 $WINDOWS_POWERSHELL_DIR\Install-Datadog.ps1 diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index eab1e718f9fee..28779481845c8 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -4,7 +4,7 @@ setup_agent_version: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] script: - - inv -e agent.version --cache-version + - inv -e agent.version --cache-version || exit $? - $S3_CP_CMD $CI_PROJECT_DIR/agent-version.cache $S3_ARTIFACTS_URI/agent-version.cache needs: [] @@ -18,15 +18,15 @@ github_rate_limit_info: script: - python3 -m pip install -r tasks/libs/requirements-github.txt datadog_api_client # Send stats for app 1 - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 # Send stats for app 2 - - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_KEY_2) || exit $?; export GITHUB_KEY_B64 - - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_ID_2) || exit $?; export GITHUB_APP_ID - - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_INSTALLATION_ID_2) || exit $?; export GITHUB_INSTALLATION_ID + - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 + - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID + - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 allow_failure: true diff --git a/.gitlab/source_test/include.yml b/.gitlab/source_test/include.yml index c71029608a40e..60666c8d4f5a1 100644 --- a/.gitlab/source_test/include.yml +++ b/.gitlab/source_test/include.yml @@ -12,5 +12,4 @@ include: - .gitlab/source_test/slack.yml - .gitlab/source_test/golang_deps_diff.yml - .gitlab/source_test/notify.yml - - .gitlab/source_test/technical_linters.yml - .gitlab/source_test/tooling_unit_tests.yml diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 9bdb505317bcb..731630dd74842 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -53,39 +53,10 @@ - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV_TOKEN) || exit $?; export CODECOV_TOKEN - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true -.linux_lint: - stage: source_test - needs: ["go_deps", "go_tools_deps"] - variables: - FLAVORS: '--flavor base' - KUBERNETES_CPU_REQUEST: 16 - KUBERNETES_MEMORY_REQUEST: 16Gi - KUBERNETES_MEMORY_LIMIT: 16Gi - script: - - !reference [.retrieve_linux_go_deps] - - !reference [.retrieve_linux_go_tools_deps] - - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "3" - - inv -e rtloader.install - - inv -e install-tools - - inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST $FLAVORS $EXTRA_OPTS - - .linux_x64: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] -tests_deb-x64-py2: - extends: - - .rtloader_tests - - .linux_x64 - rules: - - !reference [.except_disable_unit_tests] - - !reference [.except_mergequeue] - - when: on_success - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - tests_deb-x64-py3: extends: - .rtloader_tests @@ -98,11 +69,6 @@ tests_deb-x64-py3: PYTHON_RUNTIMES: '3' CONDA_ENV: ddpy3 -lint_linux-x64: - extends: - - .linux_lint - - .linux_x64 - tests_flavor_iot_deb-x64: extends: - .rtloader_tests @@ -115,13 +81,6 @@ tests_flavor_iot_deb-x64: CONDA_ENV: ddpy3 FLAVORS: '--flavor iot' -lint_flavor_iot_linux-x64: - extends: - - .linux_lint - - .linux_x64 - variables: - FLAVORS: '--flavor iot' - tests_flavor_dogstatsd_deb-x64: extends: - .rtloader_tests @@ -134,13 +93,6 @@ tests_flavor_dogstatsd_deb-x64: CONDA_ENV: ddpy3 FLAVORS: '--flavor dogstatsd' -lint_flavor_dogstatsd_linux-x64: - extends: - - .linux_lint - - .linux_x64 - variables: - FLAVORS: '--flavor dogstatsd' - tests_flavor_heroku_deb-x64: extends: - .rtloader_tests @@ -153,29 +105,6 @@ tests_flavor_heroku_deb-x64: CONDA_ENV: ddpy3 FLAVORS: '--flavor heroku' -lint_flavor_heroku_linux-x64: - extends: - - .linux_lint - - .linux_x64 - variables: - FLAVORS: '--flavor heroku' - -# Exclude systemd because it cannot succeed on Centos 6: the image doesn't have the shared object required by -# https://github.com/coreos/go-systemd/blob/c8cc474ba8655dfbdb0ac7fcc09b7faf5b643caf/sdjournal/functions.go#L46 -# This is OK because the test on systemd still runs on the debian image above -tests_rpm-x64-py2: - extends: .rtloader_tests - rules: - - !reference [.except_disable_unit_tests] - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - EXTRA_OPTS: '--build-exclude=systemd' - tests_rpm-x64-py3: extends: - .rtloader_tests @@ -190,18 +119,6 @@ tests_rpm-x64-py3: CONDA_ENV: ddpy3 EXTRA_OPTS: '--build-exclude=systemd' -tests_deb-arm64-py2: - extends: .rtloader_tests - rules: - - !reference [.except_disable_unit_tests] - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - tests_deb-arm64-py3: extends: - .rtloader_tests @@ -214,24 +131,6 @@ tests_deb-arm64-py3: PYTHON_RUNTIMES: '3' CONDA_ENV: ddpy3 -lint_linux-arm64: - extends: - - .linux_lint - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - -tests_rpm-arm64-py2: - extends: .rtloader_tests - rules: - - !reference [.except_disable_unit_tests] - - !reference [.except_mergequeue] - - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:arm64"] - variables: - PYTHON_RUNTIMES: '2' - CONDA_ENV: ddpy2 - tests_rpm-arm64-py3: extends: - .rtloader_tests @@ -263,7 +162,14 @@ new-e2e-unit-tests: extends: .linux_tests image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES tags: ["arch:amd64"] + needs: + - !reference [ .needs_new_e2e_template ] + - go_deps + - go_tools_deps before_script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + - !reference [.retrieve_linux_go_e2e_deps] # Setup AWS Credentials - mkdir -p ~/.aws - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $? diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index fe93fb8860e5b..cb8efadf9b518 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -1,3 +1,7 @@ +--- +include: + - .gitlab/common/macos.yml + tests_macos: stage: source_test # HACK: Run macOS unit tests only on full pipelines, to limit the use of macOS GitHub runners. @@ -30,94 +34,55 @@ tests_macos: reports: junit: "**/junit-out-*.xml" -lint_macos: +.tests_macos_gitlab: stage: source_test rules: - !reference [.except_mergequeue] - when: on_success - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["setup_agent_version"] - variables: - PYTHON_RUNTIMES: "3" - timeout: 6h - script: - - !reference [.setup_macos_github_app] - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - - python3 -m pip install -r tasks/libs/requirements-github.txt - - inv -e github.trigger-macos --workflow-type "lint" --datadog-agent-ref "$CI_COMMIT_SHA" --python-runtimes "$PYTHON_RUNTIMES" --version-cache "$VERSION_CACHE_CONTENT" - -.macos_gitlab: - variables: - PYTHON_RUNTIMES: "3" - # The Gitlab macOS runners are currently long runners, so we need to clean them beforehand. - before_script: - # Selecting the current Go version - - | - eval $(gimme $(cat .go-version)) - export GOPATH=$GOROOT - echo "Don't forget to regularly delete unused versions. Here are the installed versions and their memory usage on the runner:" - du -sh $HOME/.gimme/versions/* - # Remove the Python cache and env if the Python version changed - - | - PYTHON_REPO_VERSION=$(cat .python-version) - PYTHON_VERSION=$(python3 --version | awk '{print $2}' | sed 's/\.[0-9]*$//') - if [ "$PYTHON_REPO_VERSION" != "$PYTHON_VERSION" ]; then - echo "Python version $PYTHON_VERSION is different from $PYTHON_REPO_VERSION in .python-version. Cleaning the environment." - pyenv uninstall -f datadog-agent - echo "Installing Python $PYTHON_REPO_VERSION..." - pyenv virtualenv 3.11.8 datadog-agent - pyenv activate datadog-agent - else - echo "Python current version $PYTHON_VERSION is the same as .python-version. Keeping the existing environment." - fi - # Installing the job dependencies - - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-github.txt - - pyenv rehash - - inv -e rtloader.make --python-runtimes $PYTHON_RUNTIMES - - inv -e rtloader.install - - inv -e install-tools - - inv -e deps - -.lint_macos_gitlab: - stage: source_test allow_failure: true extends: .macos_gitlab - needs: ["setup_agent_version"] - script: - - inv -e linter.go --cpus 12 --timeout 60 - -.tests_macos_gitlab: - stage: source_test - allow_failure: true - rules: - !reference [.manual] - extends: .macos_gitlab - needs: ["setup_agent_version"] + needs: ["go_deps", "go_tools_deps"] variables: TEST_OUTPUT_FILE: test_output.json script: + - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] + - inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH - FAST_TESTS_FLAG="" - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi - - inv -e test --rerun-fails=2 --python-runtimes $PYTHON_RUNTIMES --race --profile --cpus 12 --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG + - inv -e test --rerun-fails=2 --python-runtimes $PYTHON_RUNTIMES --race --profile --cpus 12 --save-result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer - inv -e invoke-unit-tests + artifacts: + expire_in: 2 weeks + when: always + paths: + - $TEST_OUTPUT_FILE + - junit-*.tgz + reports: + junit: "**/junit-out-*.xml" + annotations: + - $EXTERNAL_LINKS_PATH -lint_macos_gitlab_amd64: - extends: .lint_macos_gitlab - tags: ["macos:monterey-amd64", "specific:true"] +.upload_junit_source: + - $CI_PROJECT_DIR/tools/ci/junit_upload.sh -lint_macos_gitlab_arm64: - extends: .lint_macos_gitlab - rules: - - !reference [.on_main] - - !reference [.manual] - tags: ["macos:monterey-arm64", "specific:true"] +.upload_coverage: + # Upload coverage files to Codecov. Never fail on coverage upload. + - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV_TOKEN) || exit $?; export CODECOV_TOKEN + - inv -e coverage.upload-to-codecov $COVERAGE_CACHE_FLAG || true tests_macos_gitlab_amd64: extends: .tests_macos_gitlab tags: ["macos:monterey-amd64", "specific:true"] + after_script: + - !reference [.upload_junit_source] + - !reference [.upload_coverage] tests_macos_gitlab_arm64: extends: .tests_macos_gitlab + rules: + !reference [.manual] tags: ["macos:monterey-arm64", "specific:true"] + after_script: + - !reference [.upload_junit_source] + - !reference [.upload_coverage] diff --git a/.gitlab/source_test/technical_linters.yml b/.gitlab/source_test/technical_linters.yml deleted file mode 100644 index c7759eda331bc..0000000000000 --- a/.gitlab/source_test/technical_linters.yml +++ /dev/null @@ -1,24 +0,0 @@ -lint_python: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: [] - script: - - inv -e linter.python - -lint_update_go: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: [] - script: - - inv -e linter.update-go - -validate_modules: - stage: source_test - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: [] - script: - - inv -e modules.validate - - inv -e modules.validate-used-by-otel diff --git a/.gitlab/source_test/windows.yml b/.gitlab/source_test/windows.yml index dfa1d0e2aaeb8..fa4b857241a24 100644 --- a/.gitlab/source_test/windows.yml +++ b/.gitlab/source_test/windows.yml @@ -25,6 +25,7 @@ -e CI_JOB_NAME="${CI_JOB_NAME}" -e CI_JOB_ID="${CI_JOB_ID}" -e CI_PIPELINE_ID="${CI_PIPELINE_ID}" + -e CI_PIPELINE_SOURCE="${CI_PIPELINE_SOURCE}" -e CI_REPOSITORY_URL="${CI_REPOSITORY_URL}" -e CI_COMMIT_SHA="${CI_COMMIT_SHA}" -e CI_PROJECT_NAME="${CI_PROJECT_NAME}" @@ -43,7 +44,6 @@ -e CODECOV_TOKEN="${CODECOV_TOKEN}" -e S3_PERMANENT_ARTIFACTS_URI="${S3_PERMANENT_ARTIFACTS_URI}" -e COVERAGE_CACHE_FLAG="${COVERAGE_CACHE_FLAG}" - -e GITLAB_TOKEN="${GITLAB_READ_API_TOKEN}" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\unittests.bat - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } @@ -58,46 +58,12 @@ reports: junit: "**/junit-out-*.xml" -.lint_windows_base: - stage: source_test - needs: ["go_deps", "go_tools_deps"] - tags: ["runner:windows-docker", "windowsversion:1809"] - script: - - $ErrorActionPreference = "Stop" - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - # Previously this job required only 8Gb of memory but since Go 1.20 it requires more to avoid being OOM killed. - # Each Windows VM has 32Gb of memory and contains 3 runners that can run one job at a time each (so a maximum of 3 simultaneous jobs per VM). - # Windows jobs are using either 8Gb or 16Gb of memory so we can limit memory to 16Gb on this job because even if we decided to limit to 10Gb for instance, - # it would leave 6Gb free but we could not fit another job with these 6Gb remaining. - - > - docker run --rm - -m 16384M - --storage-opt "size=50GB" - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - -e CI_PIPELINE_ID=${CI_PIPELINE_ID} - -e CI_PROJECT_NAME=${CI_PROJECT_NAME} - -e PY_RUNTIMES="$PYTHON_RUNTIMES" - -e GOMODCACHE="c:\modcache" - 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\lint.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - tests_windows-x64: extends: .tests_windows_base variables: PYTHON_RUNTIMES: 3 ARCH: "x64" -lint_windows-x64: - extends: .lint_windows_base - rules: - - !reference [.except_mergequeue] - - when: on_success - variables: - PYTHON_RUNTIMES: 3 - ARCH: "x64" - timeout: 1h30m - .tests_windows_sysprobe: stage: source_test needs: ["go_deps", "go_tools_deps"] diff --git a/.gitlab/trigger_release/trigger_release.yml b/.gitlab/trigger_release/trigger_release.yml index 2836ba48b0098..c0c67e2d50d49 100644 --- a/.gitlab/trigger_release/trigger_release.yml +++ b/.gitlab/trigger_release/trigger_release.yml @@ -18,7 +18,7 @@ script: # agent-release-management creates pipeline for both Agent 6 and Agent 7 # when triggered with major version 7 - - export RELEASE_VERSION=$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1 + - RELEASE_VERSION="$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1" || exit $?; export RELEASE_VERSION - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN - 'inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-release-management" --git-ref "main" --variable ACTION diff --git a/.go-version b/.go-version index 87b26e8b1aa0e..229a27c6f2047 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.7 +1.22.8 diff --git a/.golangci.yml b/.golangci.yml index a5860decf1f76..feb7e5a04fe49 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -582,6 +582,7 @@ linters: - gosimple # Linter for Go source code that specializes in simplifying code. - gocheckcompilerdirectives # Checks Go compiler directives syntax - pkgconfigusage # Linter for checking usage of pkgconfig inside components folder + - forbidigo # prevent usage of unwanted/deprecated symbol in the code base linters-settings: depguard: @@ -711,6 +712,18 @@ linters-settings: - name: var-naming # non-default rules: - name: duplicated-imports + forbidigo: + forbid: + - p: ^model.NewConfig.*$ + pkg: ^github.com/DataDog/datadog-agent/pkg/config/model$ + msg: use pkg/config/mock instead in tests or the config component + - p: ^setup.SetDatadog.*$ + pkg: ^github.com/DataDog/datadog-agent/pkg/config/setup$ + msg: use pkg/config/mock instead for tests or the config component + - p: ^setup.SetSystemProbe.*$ + pkg: ^github.com/DataDog/datadog-agent/pkg/config/setup$ + msg: use pkg/config/mock instead for tests or the config component + analyze-types: true custom: pkgconfigusage: type: "module" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bc80db24e44b..27f0cdfa3dec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,11 @@ repos: hooks: - id: end-of-file-fixer files: release.json +- repo: https://github.com/rhysd/actionlint + rev: v1.7.3 + hooks: + - id: actionlint + args: [ "-ignore", "shellcheck" ] - repo: local hooks: - id: protected-branches @@ -111,6 +116,14 @@ repos: files: .*gitlab.*\.yml$ pass_filenames: false stages: [pre-push] + - id: gitlab-lint-jobs-codeowners + name: gitlab-lint-jobs-codeowners + description: lint the gitlab configuration to verify jobs codeowners + entry: 'inv linter.gitlab-ci-jobs-codeowners' + language: system + require_serial: true + files: .*gitlab.*\.yml$ + pass_filenames: false - id: update-go name: update-go description: test formatting of files will allow go update diff --git a/.wwhrd.yml b/.wwhrd.yml index 5ffc72c725368..814b73843d3fe 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -45,4 +45,4 @@ exceptions: additional: # list here paths to additional licenses - golang/go: "raw.githubusercontent.com/golang/go/go1.22.7/LICENSE" + golang/go: "raw.githubusercontent.com/golang/go/go1.22.8/LICENSE" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4886cd5e1a788..abbc7762d316e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,61 @@ Release Notes ============= +.. _Release Notes_7.57.2: + +7.57.2 +====== + +.. _Release Notes_7.57.2_Prelude: + +Prelude +------- + +Release on: 2024-09-24 + + +.. _Release Notes_7.57.2_Enhancement Notes: + +Enhancement Notes +----------------- + +- Agents are now built with Go ``1.22.7``. + + +.. _Release Notes_7.57.2_Bug Fixes: + +Bug Fixes +--------- + +- Fix OOM error with cluster agent auto instrumentation by increasing default memory request from 20Mi to 100Mi. + +- Fixes a panic caused by running the Agent on readonly filesystems. The + Agent returns integration launchers and handles memory gracefully. + + +.. _Release Notes_7.57.1: + +7.57.1 +====== + +.. _Release Notes_7.57.1_Prelude: + +Prelude +------- + +Release on: 2024-09-17 + +- Please refer to the `7.57.1 tag on integrations-core `_ for the list of changes on the Core Checks + +.. _Release Notes_7.57.1_Bug Fixes: + +Bug Fixes +--------- + +- APM: When the UDS listener cannot be created on the trace-agent, the process will log the error, instead of crashing. +- Fixes memory leak caused by container check. + + .. _Release Notes_7.57.0: 7.57.0 diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index cb0fb6109e780..cf314ed6f47cc 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,9 +1,11 @@ -ARG AGENT_VERSION=nightly-ot-beta-main-jmx +ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.57.x-otel-beta-v1 # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder # Set environment variables ARG AGENT_VERSION +ARG AGENT_BRANCH ENV DEBIAN_FRONTEND=noninteractive # Set the working directory @@ -24,7 +26,7 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* # TEMP: Use github source code -RUN git clone --depth 1 https://github.com/DataDog/datadog-agent.git datadog-agent-${AGENT_VERSION} +RUN git clone --depth 1 -b "${AGENT_BRANCH}" --single-branch https://github.com/DataDog/datadog-agent.git datadog-agent-${AGENT_VERSION} # Once we have stable releases, we can use the following code to download the source code # TODO: use released agent version once we have an agent release with the otel binary @@ -77,7 +79,7 @@ RUN . venv/bin/activate && invoke collector.generate RUN . venv/bin/activate && invoke otel-agent.build # Use the final Datadog agent image -FROM datadog/agent-dev:${AGENT_VERSION} +FROM datadog/agent:${AGENT_VERSION} ARG AGENT_VERSION # Copy the built OTel agent from the builder stage COPY --from=builder /workspace/datadog-agent-${AGENT_VERSION}/bin/otel-agent/otel-agent /opt/datadog-agent/embedded/bin/otel-agent diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8630554f6b2b4..20bf49bdccb73 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -591,7 +591,6 @@ core,github.com/aws/aws-sdk-go-v2/service/sts,Apache-2.0,"Copyright 2014-2015 St core,github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go-v2/service/sts/types,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/aws,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/aws/arn,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/aws/auth/bearer,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/aws/awserr,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/aws/awsutil,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." @@ -613,9 +612,6 @@ core,github.com/aws/aws-sdk-go/aws/session,Apache-2.0,"Copyright 2014-2015 Strip core,github.com/aws/aws-sdk-go/aws/signer/v4,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/context,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/ini,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/internal/s3shared,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/internal/s3shared/arn,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/internal/s3shared/s3err,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/sdkio,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/sdkmath,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/sdkrand,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." @@ -623,24 +619,19 @@ core,github.com/aws/aws-sdk-go/internal/sdkuri,Apache-2.0,"Copyright 2014-2015 S core,github.com/aws/aws-sdk-go/internal/shareddefaults,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/strings,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/internal/sync/singleflight,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/private/checksum,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/ec2query,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/private/protocol/eventstream,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/json/jsonutil,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/jsonrpc,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/query,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/query/queryutil,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/rest,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/restjson,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/private/protocol/restxml,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/ec2,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/ec2/ec2iface,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/ecs,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/lightsail,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." -core,github.com/aws/aws-sdk-go/service/s3,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/sso,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/sso/ssoiface,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." core,github.com/aws/aws-sdk-go/service/ssooidc,Apache-2.0,"Copyright 2014-2015 Stripe, Inc. | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved." @@ -659,11 +650,13 @@ core,github.com/aws/smithy-go/endpoints,Apache-2.0,"Copyright Amazon.com, Inc. o core,github.com/aws/smithy-go/internal/sync/singleflight,BSD-3-Clause,"Copyright (c) 2009 The Go Authors. All rights reserved | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/io,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/logging,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" +core,github.com/aws/smithy-go/metrics,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/middleware,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/private/requestcompression,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/ptr,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/rand,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/time,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" +core,github.com/aws/smithy-go/tracing,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/transport/http,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/transport/http/internal/io,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" core,github.com/aws/smithy-go/waiter,Apache-2.0,"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved" @@ -676,7 +669,6 @@ core,github.com/bahlo/generic-list-go,BSD-3-Clause,Copyright (c) 2009 The Go Aut core,github.com/beevik/ntp,BSD-2-Clause,Al Cutter (AlCutter) | Andrey Smirnov (smira) | Anton Tolchanov (knyar) | Ask Bjørn Hansen (abh) | Brett Vickers (beevik) | Christopher Batey (chbatey) | Copyright © 2015-2023 Brett Vickers. All rights reserved | Leonid Evdokimov (darkk) | Meng Zhuo (mengzhuo) | Mikhail Salosin (AlphaB) | Silves-Xiang (silves-xiang) core,github.com/benbjohnson/clock,MIT,Copyright (c) 2014 Ben Johnson core,github.com/beorn7/perks/quantile,MIT,Copyright (C) 2013 Blake Mizerany -core,github.com/bgentry/go-netrc/netrc,MIT,Copyright © 2010 Fazlul Shahriar . Newer | Copyright © 2014 Blake Gentry core,github.com/bhmj/jsonslice,MIT,Copyright (c) 2018 bhmj core,github.com/bitnami/go-version/pkg/version,Apache-2.0,Copyright (c) 2023-2024 Carlos Rodríguez Hernández core,github.com/blabber/go-freebsd-sysctl/sysctl,0BSD,Copyright (c) 2014-2020 by Tobias Rehbein @@ -876,9 +868,7 @@ core,github.com/docker/docker/api/types/versions,Apache-2.0,"Copyright 2012-2017 core,github.com/docker/docker/api/types/volume,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/docker/client,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/docker/errdefs,Apache-2.0,"Copyright 2012-2017 Docker, Inc." -core,github.com/docker/docker/image/spec/specs-go/v1,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/docker/internal/multierror,Apache-2.0,"Copyright 2012-2017 Docker, Inc." -core,github.com/docker/docker/pkg/homedir,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/go-connections/nat,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/go-connections/sockets,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/docker/go-connections/tlsconfig,Apache-2.0,"Copyright 2012-2017 Docker, Inc." @@ -1044,14 +1034,6 @@ core,github.com/go-redis/redis/v8/internal/pool,BSD-2-Clause,Copyright (c) 2013 core,github.com/go-redis/redis/v8/internal/proto,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors core,github.com/go-redis/redis/v8/internal/rand,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors core,github.com/go-redis/redis/v8/internal/util,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/hashtag,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/hscan,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/pool,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/proto,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/rand,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors -core,github.com/go-redis/redis/v9/internal/util,BSD-2-Clause,Copyright (c) 2013 The github.com/go-redis/redis Authors core,github.com/go-resty/resty/v2,MIT,"Copyright (c) 2015-2023 Jeevanandam M., https://myjeeva.com " core,github.com/go-sql-driver/mysql,MPL-2.0,"Aaron Hopkins | Achille Roussel | Aidan | Alex Snast | Alexey Palazhchenko | Andrew Reid | Animesh Ray | Ariel Mashraki | Arne Hormann | Asta Xie | Barracuda Networks, Inc. | Brian Hendriks | Bulat Gaifullin | Caine Jette | Carlos Nieto | Chris Kirkland | Chris Moos | Counting Ltd. | Craig Wilson | Daemonxiao <735462752 at qq.com> | Daniel Montoya | Daniel Nichter | Daniël van Eeden | Dave Protasowski | DigitalOcean Inc. | DisposaBoy | Dolthub Inc. | Egor Smolyakov | Erwan Martin | Evan Elias | Evan Shaw | Facebook Inc. | Frederick Mayle | GitHub Inc. | Google Inc. | Gustavo Kristic | Gusted | Hajime Nakagami | Hanno Braun | Henri Yandell | Hirotaka Yamamoto | Huyiguang | ICHINOSE Shogo | INADA Naoki | Ilia Cimpoes | InfoSum Ltd. | Jacek Szwec | James Harr | Janek Vedock | Jason Ng | Jean-Yves Pellé | Jeff Hodges | Jeffrey Charles | Jennifer Purevsuren | Jerome Meyer | Jiajia Zhong | Jian Zhen | Joshua Prunier | Julien Lefevre | Julien Schmidt | Justin Li | Justin Nuß | Kamil Dziedzic | Kei Kamikawa | Kevin Malachowski | Keybase Inc. | Kieron Woodhouse | Lance Tian | Lennart Rudolph | Leonardo YongUk Kim | Linh Tran Tuan | Lion Yang | Luca Looz | Lucas Liu | Luke Scott | Lunny Xiao | Maciej Zimnoch | Michael Woolnough | Microsoft Corp. | Multiplay Ltd. | Nathanial Murphy | Nicola Peduzzi | Oliver Bone | Olivier Mengué | Paul Bonser | Paulius Lozys | Percona LLC | Peter Schultz | Phil Porada | PingCAP Inc. | Pivotal Inc. | Rebecca Chin | Reed Allman | Richard Wilkes | Robert Russell | Runrioter Wung | Samantha Frank | Santhosh Kumar Tekuri | Shattered Silicon Ltd. | Sho Iizuka | Sho Ikeda | Shuode Li | Simon J Mudd | Soroush Pour | Stan Putrya | Stanley Gunawan | Steven Hartland | Stripe Inc. | Tan Jinhua <312841925 at qq.com> | Tetsuro Aoki | Thomas Wodarek | Tim Ruffles | Tom Jenkinson | Vladimir Kovpak | Vladyslav Zhelezniak | Xiangyu Hu | Xiaobing Jiang | Xiuming Chen | Xuehong Chan | Zendesk Inc. | Zhang Xiang | Zhenye Xie | Zhixin Wen | Ziheng Lyu | copyright doctrines of fair use, fair dealing, or other | dyves labs AG | oscarzhao " core,github.com/go-viper/mapstructure/v2,MIT,Copyright (c) 2013 Mitchell Hashimoto @@ -1248,26 +1230,26 @@ core,github.com/gorilla/websocket,BSD-3-Clause,Copyright (c) 2023 The Gorilla Au core,github.com/gosnmp/gosnmp,BSD-2-Clause,Copyright 2012-2020 The GoSNMP Authors. All rights reserved. core,github.com/grafana/regexp,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,github.com/grafana/regexp/syntax,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,github.com/grpc-ecosystem/go-grpc-middleware,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/go-grpc-middleware/auth,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/go-grpc-middleware/util/metautils,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved. +core,github.com/grpc-ecosystem/go-grpc-middleware,Apache-2.0,Copyright (c) The go-grpc-middleware Authors. +core,github.com/grpc-ecosystem/go-grpc-middleware/auth,Apache-2.0,Copyright (c) The go-grpc-middleware Authors. +core,github.com/grpc-ecosystem/go-grpc-middleware/util/metautils,Apache-2.0,Copyright (c) The go-grpc-middleware Authors. core,github.com/grpc-ecosystem/go-grpc-prometheus,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/codegenerator,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/internal,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/internal/casing,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/generator,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/httprule,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/internal/gengateway,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/genswagger,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/runtime,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/utilities,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/v2/runtime,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. -core,github.com/grpc-ecosystem/grpc-gateway/v2/utilities,BSD-3-Clause,Copyright 2016 Michal Witkowski. All Rights Reserved. +core,github.com/grpc-ecosystem/grpc-gateway/codegenerator,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/internal,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/internal/casing,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc. | Copyright 2010 The Go Authors. All rights reserved" +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/generator,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/httprule,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/internal/gengateway,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/genswagger,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/runtime,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/utilities,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc | Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/v2/runtime,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc | Copyright (c) 2015, Gengo, Inc." +core,github.com/grpc-ecosystem/grpc-gateway/v2/utilities,BSD-3-Clause,"Copyright (c) 2015, Gengo, Inc | Copyright (c) 2015, Gengo, Inc." core,github.com/h2non/filetype,MIT,Copyright (c) Tomas Aparicio core,github.com/h2non/filetype/matchers,MIT,Copyright (c) Tomas Aparicio core,github.com/h2non/filetype/matchers/isobmff,MIT,Copyright (c) Tomas Aparicio @@ -1276,14 +1258,11 @@ core,github.com/hashicorp/consul/api,MPL-2.0,"Copyright © 2014-2018 HashiCorp, core,github.com/hashicorp/cronexpr,Apache-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/errwrap,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-cleanhttp,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" -core,github.com/hashicorp/go-getter,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" -core,github.com/hashicorp/go-getter/helper/url,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-hclog,MIT,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-immutable-radix,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-multierror,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-retryablehttp,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-rootcerts,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" -core,github.com/hashicorp/go-safetemp,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-secure-stdlib/parseutil,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-secure-stdlib/strutil,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" core,github.com/hashicorp/go-sockaddr,MPL-2.0,"Copyright © 2014-2018 HashiCorp, Inc" @@ -1471,7 +1450,6 @@ core,github.com/microsoft/go-rustaudit,MIT,Copyright (c) Microsoft Corporation core,github.com/miekg/dns,BSD-3-Clause,"Alex A. Skinner | Alex Sergeyev | Andrew Tunnell-Jones | Ask Bjørn Hansen | Copyright (c) 2009, The Go Authors. Extensions copyright (c) 2011, Miek Gieben | Copyright 2009 The Go Authors. All rights reserved. Use of this source code | Copyright 2011 Miek Gieben. All rights reserved. Use of this source code is | Copyright 2014 CloudFlare. All rights reserved. Use of this source code is | Dave Cheney | Dusty Wilson | James Hartig | Marek Majkowski | Miek Gieben | Omri Bahumi | Peter van Dijk | copyright (c) 2011 Miek Gieben" core,github.com/mitchellh/copystructure,MIT,Copyright (c) 2014 Mitchell Hashimoto core,github.com/mitchellh/go-homedir,MIT,Copyright (c) 2013 Mitchell Hashimoto -core,github.com/mitchellh/go-testing-interface,MIT,Copyright (c) 2016 Mitchell Hashimoto core,github.com/mitchellh/hashstructure/v2,MIT,Copyright (c) 2016 Mitchell Hashimoto core,github.com/mitchellh/mapstructure,MIT,Copyright (c) 2013 Mitchell Hashimoto core,github.com/mitchellh/reflectwalk,MIT,Copyright (c) 2013 Mitchell Hashimoto @@ -1482,6 +1460,7 @@ core,github.com/moby/buildkit/frontend/dockerfile/parser,Apache-2.0,"Aaron L. Xu core,github.com/moby/buildkit/frontend/dockerfile/shell,Apache-2.0,"Aaron L. Xu | Aaron Lehmann | Aaron Lehmann | Abdur Rehman | Addam Hardy | Adrian Plata | Aidan Hobson Sayers | Akihiro Suda | Alan Fregtman <941331+darkvertex@users.noreply.github.com> | Alex Couture-Beil | Alex Mayer | Alex Suraci | Alexander Morozov | Alexis Murzeau | Alice Frosi | Allen Sun | Amen Belayneh | Anca Iordache | Anda Xu | Anders F Björklund | Andrea Bolognani | Andrea Luzzardi | Andrew Chang | Andrey Smirnov | Andy Alt | Andy Caldwell | Ankush Agarwal | Anthony Sottile | Anurag Goel | Anusha Ragunathan | Arnaud Bailly | Avi Deitcher | Bastiaan Bakker | Ben Longo | Bertrand Paquet | Bin Liu | Brandon Mitchell | Brian Goff | Ce Gao | Chaerim Yeo | Changwei Ge | Chanhun Jeong | ChaosGramer | Charles Chan | Charles Korn | Charles Law | Chenbin | Chris Goller | Chris McKinnel | Christian Höltje | Christian Weichel | Ciro S. Costa | Claudiu Belu | Colin Chartier | Corey Larson | Cory Bennett | Cory Snider | CrazyMax | Csaba Apagyi | Dan Duvall | Daniel Cassidy | Daniel Nephin | Darren Shepherd | Dave Chen | Dave Henderson | Dave Tucker | David Calavera | David Dooling | David Gageot | David Karlsson | Davis Schirmer | Dennis Chen | Derek McGowan | Dharmit Shah | Ding Fei | Doug Davis | Edgar Lee | Eli Uriegas | Elias Faxö | Eng Zer Jun | Eric Engestrom | Erik Sipsma | Fernando Miguel | Fiona Klute | Foysal Iqbal | Fred Cox | Frieder Bluemle | Gabriel | Gabriel Adrian Samfira | Gaetan de Villele | Gahl Saraf | George | Govind Rai | Grant Reaber | Guilhem C | Hans van den Bogert | Hao Hu | Hector S | Helen Xie | Himanshu Pandey | Hiromu Nakamura | HowJMay | Hugo Santos | Ian Campbell | Ilya Dmitrichenko | Iskander (Alex) Sharipov | Jacob Gillespie | Jacob MacElroy | Jean-Pierre Huynh | Jeffrey Huang | Jesse Rittner | Jessica Frazelle | Jitender Kumar | John Howard | John Maguire | John Mulhausen | John Tims | Jon Zeolla | Jonathan Azoff | Jonathan Giannuzzi | Jonathan Stoppani | Jonny Stoten | JordanGoasdoue | Julian Goede | Justas Brazauskas | Justin Chadwell | Justin Cormack | Justin Garrison | Jörg Franke <359489+NewJorg@users.noreply.github.com> | Kang, Matthew | Kees Cook | Kevin Burke | Kir Kolyshkin | Kohei Tokunaga | Koichi Shiraishi | Kris-Mikael Krister | Kunal Kushwaha | Kyle | Lajos Papp | Levi Harrison | Lu Jingxiao | Luca Visentin | Maciej Kalisz | Madhav Puri | Manu Gupta | Marcus Comstedt | Mark Gordon | Marko Kohtala | Mary Anthony | Matias Insaurralde | Matt Kang | Matt Rickard | Maxime Lagresle | Michael Crosby | Michael Friis | Michael Irwin | Miguel Ángel Jimeno | Mihai Borobocea | Mike Brown | Mikhail Vasin | Misty Stanley-Jones | Miyachi Katsuya | Morgan Bauer | Morlay | Nao YONASHIRO | Natasha Jarus | Nathan Sullivan | Nick Miyake | Nick Santos | Nikhil Pandeti | Noel Georgi <18496730+frezbo@users.noreply.github.com> | Oliver Bristow | Omer Duchovne <79370724+od-cyera@users.noreply.github.com> | Omer Mizrahi | Ondrej Fabry | Otto Kekäläinen | Pablo Chico de Guzman | Patrick Hemmer | Patrick Lang | Patrick Van Stee | Paul ""TBBle"" Hampson | Paweł Gronowski | Peter Dave Hello | Petr Fedchenkov | Phil Estes | Pierre Fenoll | Pranav Pandit | Pratik Raj | Prayag Verma | Qiang Huang | Remy Suen | Ri Xu | Rob Taylor | Robert Estelle | Rubens Figueiredo | Sam Whited | Sascha Schwarze | Sean P. Kane | Sebastiaan van Stijn | Seiya Miyata | Serhat Gülçiçek | Sertac Ozercan | Shev Yan | Shijiang Wei | Shingo Omura | Shiwei Zhang | Siebe Schaap | Silvin Lubecki <31478878+silvin-lubecki@users.noreply.github.com> | Simon Ferquel | Slava Semushin | Solomon Hykes | Stefan Scherer | Stefan Weil | StefanSchoof | Stepan Blyshchak | Steve Lohr | Sven Dowideit | Takuya Noguchi | Thomas Leonard | Thomas Riccardi | Thomas Shaw | Tianon Gravi | Tibor Vass | Tiffany Jernigan | Tim Waugh | Tim Wraight | Tino Rusch | Tobias Klauser | Tomas Tomecek | Tomasz Kopczynski | Tomohiro Kusumoto | Troels Liebe Bentsen | Tõnis Tiigi | Valentin Lorentz | Vasek - Tom C | Victor Vieux | Victoria Bialas | Vincent Demeester | Vlad A. Ionescu | Vladislav Ivanov | Wang Yumu <37442693@qq.com> | Wei Fu | Wei Zhang | Xiaofan Zhang | Ximo Guanter | Yamazaki Masashi | Yan Song | Yong Tang | Yuichiro Kaneko | Yurii Rashkovskii | Zach Badgett | Ziv Tsarfati | a-palchikov | coryb | dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> | dito | eyherabh | f0 | genglu.gl | ggjulio | jgeiger | jlecordier | joey | jroenf | kevinmeredith | l00397676 | lalyos | liwenqi | lixiaobing10051267 | lomot | masibw | mikelinjie <294893458@qq.com> | msg | pieterdd | squeegels <1674195+squeegels@users.noreply.github.com> | sunchunming | wingkwong | zhangwenlong | 岁丰 | 沈陵 | 郑泽宇 " core,github.com/moby/buildkit/util/stack,Apache-2.0,"Aaron L. Xu | Aaron Lehmann | Aaron Lehmann | Abdur Rehman | Addam Hardy | Adrian Plata | Aidan Hobson Sayers | Akihiro Suda | Alan Fregtman <941331+darkvertex@users.noreply.github.com> | Alex Couture-Beil | Alex Mayer | Alex Suraci | Alexander Morozov | Alexis Murzeau | Alice Frosi | Allen Sun | Amen Belayneh | Anca Iordache | Anda Xu | Anders F Björklund | Andrea Bolognani | Andrea Luzzardi | Andrew Chang | Andrey Smirnov | Andy Alt | Andy Caldwell | Ankush Agarwal | Anthony Sottile | Anurag Goel | Anusha Ragunathan | Arnaud Bailly | Avi Deitcher | Bastiaan Bakker | Ben Longo | Bertrand Paquet | Bin Liu | Brandon Mitchell | Brian Goff | Ce Gao | Chaerim Yeo | Changwei Ge | Chanhun Jeong | ChaosGramer | Charles Chan | Charles Korn | Charles Law | Chenbin | Chris Goller | Chris McKinnel | Christian Höltje | Christian Weichel | Ciro S. Costa | Claudiu Belu | Colin Chartier | Corey Larson | Cory Bennett | Cory Snider | CrazyMax | Csaba Apagyi | Dan Duvall | Daniel Cassidy | Daniel Nephin | Darren Shepherd | Dave Chen | Dave Henderson | Dave Tucker | David Calavera | David Dooling | David Gageot | David Karlsson | Davis Schirmer | Dennis Chen | Derek McGowan | Dharmit Shah | Ding Fei | Doug Davis | Edgar Lee | Eli Uriegas | Elias Faxö | Eng Zer Jun | Eric Engestrom | Erik Sipsma | Fernando Miguel | Fiona Klute | Foysal Iqbal | Fred Cox | Frieder Bluemle | Gabriel | Gabriel Adrian Samfira | Gaetan de Villele | Gahl Saraf | George | Govind Rai | Grant Reaber | Guilhem C | Hans van den Bogert | Hao Hu | Hector S | Helen Xie | Himanshu Pandey | Hiromu Nakamura | HowJMay | Hugo Santos | Ian Campbell | Ilya Dmitrichenko | Iskander (Alex) Sharipov | Jacob Gillespie | Jacob MacElroy | Jean-Pierre Huynh | Jeffrey Huang | Jesse Rittner | Jessica Frazelle | Jitender Kumar | John Howard | John Maguire | John Mulhausen | John Tims | Jon Zeolla | Jonathan Azoff | Jonathan Giannuzzi | Jonathan Stoppani | Jonny Stoten | JordanGoasdoue | Julian Goede | Justas Brazauskas | Justin Chadwell | Justin Cormack | Justin Garrison | Jörg Franke <359489+NewJorg@users.noreply.github.com> | Kang, Matthew | Kees Cook | Kevin Burke | Kir Kolyshkin | Kohei Tokunaga | Koichi Shiraishi | Kris-Mikael Krister | Kunal Kushwaha | Kyle | Lajos Papp | Levi Harrison | Lu Jingxiao | Luca Visentin | Maciej Kalisz | Madhav Puri | Manu Gupta | Marcus Comstedt | Mark Gordon | Marko Kohtala | Mary Anthony | Matias Insaurralde | Matt Kang | Matt Rickard | Maxime Lagresle | Michael Crosby | Michael Friis | Michael Irwin | Miguel Ángel Jimeno | Mihai Borobocea | Mike Brown | Mikhail Vasin | Misty Stanley-Jones | Miyachi Katsuya | Morgan Bauer | Morlay | Nao YONASHIRO | Natasha Jarus | Nathan Sullivan | Nick Miyake | Nick Santos | Nikhil Pandeti | Noel Georgi <18496730+frezbo@users.noreply.github.com> | Oliver Bristow | Omer Duchovne <79370724+od-cyera@users.noreply.github.com> | Omer Mizrahi | Ondrej Fabry | Otto Kekäläinen | Pablo Chico de Guzman | Patrick Hemmer | Patrick Lang | Patrick Van Stee | Paul ""TBBle"" Hampson | Paweł Gronowski | Peter Dave Hello | Petr Fedchenkov | Phil Estes | Pierre Fenoll | Pranav Pandit | Pratik Raj | Prayag Verma | Qiang Huang | Remy Suen | Ri Xu | Rob Taylor | Robert Estelle | Rubens Figueiredo | Sam Whited | Sascha Schwarze | Sean P. Kane | Sebastiaan van Stijn | Seiya Miyata | Serhat Gülçiçek | Sertac Ozercan | Shev Yan | Shijiang Wei | Shingo Omura | Shiwei Zhang | Siebe Schaap | Silvin Lubecki <31478878+silvin-lubecki@users.noreply.github.com> | Simon Ferquel | Slava Semushin | Solomon Hykes | Stefan Scherer | Stefan Weil | StefanSchoof | Stepan Blyshchak | Steve Lohr | Sven Dowideit | Takuya Noguchi | Thomas Leonard | Thomas Riccardi | Thomas Shaw | Tianon Gravi | Tibor Vass | Tiffany Jernigan | Tim Waugh | Tim Wraight | Tino Rusch | Tobias Klauser | Tomas Tomecek | Tomasz Kopczynski | Tomohiro Kusumoto | Troels Liebe Bentsen | Tõnis Tiigi | Valentin Lorentz | Vasek - Tom C | Victor Vieux | Victoria Bialas | Vincent Demeester | Vlad A. Ionescu | Vladislav Ivanov | Wang Yumu <37442693@qq.com> | Wei Fu | Wei Zhang | Xiaofan Zhang | Ximo Guanter | Yamazaki Masashi | Yan Song | Yong Tang | Yuichiro Kaneko | Yurii Rashkovskii | Zach Badgett | Ziv Tsarfati | a-palchikov | coryb | dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> | dito | eyherabh | f0 | genglu.gl | ggjulio | jgeiger | jlecordier | joey | jroenf | kevinmeredith | l00397676 | lalyos | liwenqi | lixiaobing10051267 | lomot | masibw | mikelinjie <294893458@qq.com> | msg | pieterdd | squeegels <1674195+squeegels@users.noreply.github.com> | sunchunming | wingkwong | zhangwenlong | 岁丰 | 沈陵 | 郑泽宇 " core,github.com/moby/buildkit/util/suggest,Apache-2.0,"Aaron L. Xu | Aaron Lehmann | Aaron Lehmann | Abdur Rehman | Addam Hardy | Adrian Plata | Aidan Hobson Sayers | Akihiro Suda | Alan Fregtman <941331+darkvertex@users.noreply.github.com> | Alex Couture-Beil | Alex Mayer | Alex Suraci | Alexander Morozov | Alexis Murzeau | Alice Frosi | Allen Sun | Amen Belayneh | Anca Iordache | Anda Xu | Anders F Björklund | Andrea Bolognani | Andrea Luzzardi | Andrew Chang | Andrey Smirnov | Andy Alt | Andy Caldwell | Ankush Agarwal | Anthony Sottile | Anurag Goel | Anusha Ragunathan | Arnaud Bailly | Avi Deitcher | Bastiaan Bakker | Ben Longo | Bertrand Paquet | Bin Liu | Brandon Mitchell | Brian Goff | Ce Gao | Chaerim Yeo | Changwei Ge | Chanhun Jeong | ChaosGramer | Charles Chan | Charles Korn | Charles Law | Chenbin | Chris Goller | Chris McKinnel | Christian Höltje | Christian Weichel | Ciro S. Costa | Claudiu Belu | Colin Chartier | Corey Larson | Cory Bennett | Cory Snider | CrazyMax | Csaba Apagyi | Dan Duvall | Daniel Cassidy | Daniel Nephin | Darren Shepherd | Dave Chen | Dave Henderson | Dave Tucker | David Calavera | David Dooling | David Gageot | David Karlsson | Davis Schirmer | Dennis Chen | Derek McGowan | Dharmit Shah | Ding Fei | Doug Davis | Edgar Lee | Eli Uriegas | Elias Faxö | Eng Zer Jun | Eric Engestrom | Erik Sipsma | Fernando Miguel | Fiona Klute | Foysal Iqbal | Fred Cox | Frieder Bluemle | Gabriel | Gabriel Adrian Samfira | Gaetan de Villele | Gahl Saraf | George | Govind Rai | Grant Reaber | Guilhem C | Hans van den Bogert | Hao Hu | Hector S | Helen Xie | Himanshu Pandey | Hiromu Nakamura | HowJMay | Hugo Santos | Ian Campbell | Ilya Dmitrichenko | Iskander (Alex) Sharipov | Jacob Gillespie | Jacob MacElroy | Jean-Pierre Huynh | Jeffrey Huang | Jesse Rittner | Jessica Frazelle | Jitender Kumar | John Howard | John Maguire | John Mulhausen | John Tims | Jon Zeolla | Jonathan Azoff | Jonathan Giannuzzi | Jonathan Stoppani | Jonny Stoten | JordanGoasdoue | Julian Goede | Justas Brazauskas | Justin Chadwell | Justin Cormack | Justin Garrison | Jörg Franke <359489+NewJorg@users.noreply.github.com> | Kang, Matthew | Kees Cook | Kevin Burke | Kir Kolyshkin | Kohei Tokunaga | Koichi Shiraishi | Kris-Mikael Krister | Kunal Kushwaha | Kyle | Lajos Papp | Levi Harrison | Lu Jingxiao | Luca Visentin | Maciej Kalisz | Madhav Puri | Manu Gupta | Marcus Comstedt | Mark Gordon | Marko Kohtala | Mary Anthony | Matias Insaurralde | Matt Kang | Matt Rickard | Maxime Lagresle | Michael Crosby | Michael Friis | Michael Irwin | Miguel Ángel Jimeno | Mihai Borobocea | Mike Brown | Mikhail Vasin | Misty Stanley-Jones | Miyachi Katsuya | Morgan Bauer | Morlay | Nao YONASHIRO | Natasha Jarus | Nathan Sullivan | Nick Miyake | Nick Santos | Nikhil Pandeti | Noel Georgi <18496730+frezbo@users.noreply.github.com> | Oliver Bristow | Omer Duchovne <79370724+od-cyera@users.noreply.github.com> | Omer Mizrahi | Ondrej Fabry | Otto Kekäläinen | Pablo Chico de Guzman | Patrick Hemmer | Patrick Lang | Patrick Van Stee | Paul ""TBBle"" Hampson | Paweł Gronowski | Peter Dave Hello | Petr Fedchenkov | Phil Estes | Pierre Fenoll | Pranav Pandit | Pratik Raj | Prayag Verma | Qiang Huang | Remy Suen | Ri Xu | Rob Taylor | Robert Estelle | Rubens Figueiredo | Sam Whited | Sascha Schwarze | Sean P. Kane | Sebastiaan van Stijn | Seiya Miyata | Serhat Gülçiçek | Sertac Ozercan | Shev Yan | Shijiang Wei | Shingo Omura | Shiwei Zhang | Siebe Schaap | Silvin Lubecki <31478878+silvin-lubecki@users.noreply.github.com> | Simon Ferquel | Slava Semushin | Solomon Hykes | Stefan Scherer | Stefan Weil | StefanSchoof | Stepan Blyshchak | Steve Lohr | Sven Dowideit | Takuya Noguchi | Thomas Leonard | Thomas Riccardi | Thomas Shaw | Tianon Gravi | Tibor Vass | Tiffany Jernigan | Tim Waugh | Tim Wraight | Tino Rusch | Tobias Klauser | Tomas Tomecek | Tomasz Kopczynski | Tomohiro Kusumoto | Troels Liebe Bentsen | Tõnis Tiigi | Valentin Lorentz | Vasek - Tom C | Victor Vieux | Victoria Bialas | Vincent Demeester | Vlad A. Ionescu | Vladislav Ivanov | Wang Yumu <37442693@qq.com> | Wei Fu | Wei Zhang | Xiaofan Zhang | Ximo Guanter | Yamazaki Masashi | Yan Song | Yong Tang | Yuichiro Kaneko | Yurii Rashkovskii | Zach Badgett | Ziv Tsarfati | a-palchikov | coryb | dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> | dito | eyherabh | f0 | genglu.gl | ggjulio | jgeiger | jlecordier | joey | jroenf | kevinmeredith | l00397676 | lalyos | liwenqi | lixiaobing10051267 | lomot | masibw | mikelinjie <294893458@qq.com> | msg | pieterdd | squeegels <1674195+squeegels@users.noreply.github.com> | sunchunming | wingkwong | zhangwenlong | 岁丰 | 沈陵 | 郑泽宇 " +core,github.com/moby/docker-image-spec/specs-go/v1,Apache-2.0,"Copyright 2012-2017 Docker, Inc." core,github.com/moby/locker,Apache-2.0,"Copyright 2013-2018 Docker, Inc" core,github.com/moby/spdystream,Apache-2.0,"Copyright 2013-2021 Docker, inc. Released under the [Apache 2.0 license](LICENSE) | Copyright 2014-2021 Docker Inc" core,github.com/moby/spdystream/spdy,Apache-2.0,"Copyright 2013-2021 Docker, inc. Released under the [Apache 2.0 license](LICENSE) | Copyright 2014-2021 Docker Inc" @@ -1882,9 +1861,9 @@ core,github.com/pierrec/lz4/v4/internal/lz4block,BSD-3-Clause,"Copyright (c) 201 core,github.com/pierrec/lz4/v4/internal/lz4errors,BSD-3-Clause,"Copyright (c) 2015, Pierre Curto" core,github.com/pierrec/lz4/v4/internal/lz4stream,BSD-3-Clause,"Copyright (c) 2015, Pierre Curto" core,github.com/pierrec/lz4/v4/internal/xxh32,BSD-3-Clause,"Copyright (c) 2015, Pierre Curto" -core,github.com/pjbgf/sha1cd,Apache-2.0,Copyright 2023 pjbgf -core,github.com/pjbgf/sha1cd/internal,Apache-2.0,Copyright 2023 pjbgf -core,github.com/pjbgf/sha1cd/ubc,Apache-2.0,Copyright 2023 pjbgf +core,github.com/pjbgf/sha1cd,Apache-2.0,"Copyright 2009 The Go Authors. All rights reserved. | Copyright 2017 Marc Stevens , Dan Shumow | Copyright 2022 Paulo Gomes | Copyright 2023 pjbgf" +core,github.com/pjbgf/sha1cd/internal,Apache-2.0,"Copyright 2009 The Go Authors. All rights reserved. | Copyright 2017 Marc Stevens , Dan Shumow | Copyright 2022 Paulo Gomes | Copyright 2023 pjbgf" +core,github.com/pjbgf/sha1cd/ubc,Apache-2.0,"Copyright 2009 The Go Authors. All rights reserved. | Copyright 2017 Marc Stevens , Dan Shumow | Copyright 2022 Paulo Gomes | Copyright 2023 pjbgf" core,github.com/pkg/browser,BSD-2-Clause,"Copyright (c) 2014, Dave Cheney " core,github.com/pkg/errors,BSD-2-Clause,"Copyright (c) 2015, Dave Cheney " core,github.com/planetscale/vtprotobuf/protohelpers,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright (c) 2018 The Go Authors. All rights reserved | Copyright (c) 2021, PlanetScale Inc. All rights reserved" @@ -1968,6 +1947,14 @@ core,github.com/prometheus/prometheus/util/pool,Apache-2.0,Copyright 2012-2015 T core,github.com/prometheus/prometheus/util/strutil,Apache-2.0,Copyright 2012-2015 The Prometheus Authors core,github.com/prometheus/prometheus/util/treecache,Apache-2.0,Copyright 2012-2015 The Prometheus Authors core,github.com/rcrowley/go-metrics,BSD-2-Clause-Views,Copyright 2012 Richard Crowley. All rights reserved +core,github.com/redis/go-redis/v9,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/hashtag,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/hscan,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/pool,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/proto,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/rand,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors +core,github.com/redis/go-redis/v9/internal/util,BSD-2-Clause,Copyright (c) 2013 The github.com/redis/go-redis Authors core,github.com/remyoudompheng/bigfft,BSD-3-Clause,Copyright (c) 2012 The Go Authors. All rights reserved core,github.com/richardartoul/molecule,MIT,Copyright (c) 2020 Richard Artoul core,github.com/richardartoul/molecule/src/codec,Apache-2.0,Copyright (c) 2020 Richard Artoul @@ -2186,7 +2173,6 @@ core,github.com/twmb/franz-go/pkg/kversion,BSD-3-Clause,"Copyright 2020, Travis core,github.com/twmb/franz-go/pkg/sasl,BSD-3-Clause,"Copyright 2020, Travis Bischel" core,github.com/twmb/murmur3,BSD-3-Clause,"Copyright 2013, Sébastien Paolacci | Copyright 2018, Travis Bischel" core,github.com/ugorji/go/codec,MIT,Copyright (c) 2012-2020 Ugorji Nwoke -core,github.com/ulikunitz/xz,BSD-3-Clause,Copyright (c) 2014-2022 Ulrich Kunitz core,github.com/ulikunitz/xz/internal/hash,BSD-3-Clause,Copyright (c) 2014-2022 Ulrich Kunitz core,github.com/ulikunitz/xz/internal/xlog,BSD-3-Clause,Copyright (c) 2014-2022 Ulrich Kunitz core,github.com/ulikunitz/xz/lzma,BSD-3-Clause,Copyright (c) 2014-2022 Ulrich Kunitz @@ -2450,6 +2436,7 @@ core,go.opentelemetry.io/contrib/config,Apache-2.0,Copyright The OpenTelemetry A core,go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp,Apache-2.0,Copyright The OpenTelemetry Authors +core,go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconvutil,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/contrib/propagators/b3,Apache-2.0,Copyright The OpenTelemetry Authors @@ -2519,7 +2506,6 @@ core,go.opentelemetry.io/otel/semconv/v1.17.0,Apache-2.0,Copyright The OpenTelem core,go.opentelemetry.io/otel/semconv/v1.17.0/httpconv,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/otel/semconv/v1.20.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/otel/semconv/v1.21.0,Apache-2.0,Copyright The OpenTelemetry Authors -core,go.opentelemetry.io/otel/semconv/v1.24.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/otel/semconv/v1.25.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/otel/semconv/v1.26.0,Apache-2.0,Copyright The OpenTelemetry Authors core,go.opentelemetry.io/otel/trace,Apache-2.0,Copyright The OpenTelemetry Authors @@ -2630,17 +2616,17 @@ core,golang.org/x/net/proxy,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/net/publicsuffix,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/net/trace,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/net/websocket,BSD-3-Clause,Copyright 2009 The Go Authors -core,golang.org/x/oauth2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/authhandler,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/clientcredentials,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/google,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/google/externalaccount,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/google/internal/externalaccountauthorizeduser,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/google/internal/impersonate,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/google/internal/stsexchange,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/internal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/jws,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/oauth2/jwt,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +core,golang.org/x/oauth2,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/authhandler,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/clientcredentials,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/google,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/google/externalaccount,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/google/internal/externalaccountauthorizeduser,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/google/internal/impersonate,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/google/internal/stsexchange,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/internal,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/jws,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/oauth2/jwt,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/sync/errgroup,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/sync/semaphore,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/sync/singleflight,BSD-3-Clause,Copyright 2009 The Go Authors @@ -2740,16 +2726,16 @@ core,google.golang.org/api/transport,BSD-3-Clause,Copyright (c) 2011 Google Inc. core,google.golang.org/api/transport/grpc,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved. core,google.golang.org/api/transport/http,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved. core,google.golang.org/api/transport/http/internal/propagation,BSD-3-Clause,Copyright (c) 2011 Google Inc. All rights reserved. -core,google.golang.org/genproto/googleapis/api,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/api/annotations,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/api/expr/v1alpha1,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/api/httpbody,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/rpc/code,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/rpc/errdetails,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/rpc/status,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/type/date,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/googleapis/type/expr,Apache-2.0,Copyright 2015 Google LLC -core,google.golang.org/genproto/protobuf/field_mask,Apache-2.0,Copyright 2015 Google LLC +core,google.golang.org/genproto/googleapis/api,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/api/annotations,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/api/expr/v1alpha1,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/api/httpbody,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/rpc/code,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/rpc/errdetails,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/rpc/status,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/type/date,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/googleapis/type/expr,Apache-2.0,Copyright 2020 Google LLC +core,google.golang.org/genproto/protobuf/field_mask,Apache-2.0,Copyright 2020 Google LLC core,google.golang.org/grpc,Apache-2.0,Copyright 2014 gRPC authors. core,google.golang.org/grpc/attributes,Apache-2.0,Copyright 2014 gRPC authors. core,google.golang.org/grpc/backoff,Apache-2.0,Copyright 2014 gRPC authors. diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index d20065de8d2f8..8ff855aeca19f 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -23,6 +23,7 @@ import ( confad "github.com/DataDog/datadog-agent/pkg/config/autodiscovery" pkgconfigenv "github.com/DataDog/datadog-agent/pkg/config/env" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/config/structure" "github.com/DataDog/datadog-agent/pkg/util/jsonquery" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -62,7 +63,7 @@ func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, // Register additional configuration providers var configProviders []pkgconfigsetup.ConfigurationProviders var uniqueConfigProviders map[string]pkgconfigsetup.ConfigurationProviders - err := pkgconfigsetup.Datadog().UnmarshalKey("config_providers", &configProviders) + err := structure.UnmarshalKey(pkgconfigsetup.Datadog(), "config_providers", &configProviders) if err == nil { uniqueConfigProviders = make(map[string]pkgconfigsetup.ConfigurationProviders, len(configProviders)+len(extraEnvProviders)+len(configProviders)) @@ -125,7 +126,7 @@ func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, } var listeners []pkgconfigsetup.Listeners - err = pkgconfigsetup.Datadog().UnmarshalKey("listeners", &listeners) + err = structure.UnmarshalKey(pkgconfigsetup.Datadog(), "listeners", &listeners) if err == nil { // Add extra listeners for _, name := range pkgconfigsetup.Datadog().GetStringSlice("extra_listeners") { diff --git a/cmd/agent/common/common.go b/cmd/agent/common/common.go index 272fed027007f..285cb3a8b5c95 100644 --- a/cmd/agent/common/common.go +++ b/cmd/agent/common/common.go @@ -13,11 +13,11 @@ import ( "net/http" "path/filepath" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/settings" settingshttp "github.com/DataDog/datadog-agent/pkg/config/settings/http" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/version" ) @@ -26,9 +26,9 @@ import ( func GetPythonPaths() []string { // wheels install in default site - already in sys.path; takes precedence over any additional location return []string{ - path.GetDistPath(), // common modules are shipped in the dist path directly or under the "checks/" sub-dir - path.PyChecksPath, // integrations-core legacy checks - filepath.Join(path.GetDistPath(), "checks.d"), // custom checks in the "checks.d/" sub-dir of the dist path + defaultpaths.GetDistPath(), // common modules are shipped in the dist path directly or under the "checks/" sub-dir + defaultpaths.PyChecksPath, // integrations-core legacy checks + filepath.Join(defaultpaths.GetDistPath(), "checks.d"), // custom checks in the "checks.d/" sub-dir of the dist path pkgconfigsetup.Datadog().GetString("additional_checksd"), // custom checks, least precedent check location } } diff --git a/cmd/agent/common/common_windows.go b/cmd/agent/common/common_windows.go index 9bcf4cf48b75a..852d4d77d5ccd 100644 --- a/cmd/agent/common/common_windows.go +++ b/cmd/agent/common/common_windows.go @@ -9,8 +9,8 @@ import ( "os" "path/filepath" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/winutil" "github.com/DataDog/datadog-agent/pkg/util/winutil/messagestrings" @@ -24,7 +24,7 @@ const ServiceName = "DatadogAgent" func init() { _, err := winutil.GetProgramDataDir() if err != nil { - winutil.LogEventViewer(ServiceName, messagestrings.MSG_WARNING_PROGRAMDATA_ERROR, path.DefaultConfPath) + winutil.LogEventViewer(ServiceName, messagestrings.MSG_WARNING_PROGRAMDATA_ERROR, defaultpaths.ConfPath) } } @@ -43,12 +43,12 @@ func EnableLoggingToFile() { // CheckAndUpgradeConfig checks to see if there's an old datadog.conf, and if // datadog.yaml is either missing or incomplete (no API key). If so, upgrade it func CheckAndUpgradeConfig() error { - datadogConfPath := filepath.Join(path.DefaultConfPath, "datadog.conf") + datadogConfPath := filepath.Join(defaultpaths.ConfPath, "datadog.conf") if _, err := os.Stat(datadogConfPath); os.IsNotExist(err) { log.Debug("Previous config file not found, not upgrading") return nil } - pkgconfigsetup.Datadog().AddConfigPath(path.DefaultConfPath) + pkgconfigsetup.Datadog().AddConfigPath(defaultpaths.ConfPath) _, err := pkgconfigsetup.LoadWithoutSecret(pkgconfigsetup.Datadog(), nil) if err == nil { // was able to read config, check for api key @@ -57,7 +57,7 @@ func CheckAndUpgradeConfig() error { return nil } } - err = ImportConfig(path.DefaultConfPath, path.DefaultConfPath, false) + err = ImportConfig(defaultpaths.ConfPath, defaultpaths.ConfPath, false) if err != nil { winutil.LogEventViewer(ServiceName, messagestrings.MSG_WARN_CONFIGUPGRADE_FAILED, err.Error()) return err diff --git a/cmd/agent/common/loader.go b/cmd/agent/common/loader.go index 57ae37a8356c4..890d7f7d422ee 100644 --- a/cmd/agent/common/loader.go +++ b/cmd/agent/common/loader.go @@ -10,12 +10,12 @@ import ( "fmt" "path/filepath" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/secrets" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/sbom/scanner" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -41,7 +41,7 @@ func GetWorkloadmetaInit() workloadmeta.InitHelper { func LoadComponents(_ secrets.Component, wmeta workloadmeta.Component, ac autodiscovery.Component, confdPath string) { confSearchPaths := []string{ confdPath, - filepath.Join(path.GetDistPath(), "conf.d"), + filepath.Join(defaultpaths.GetDistPath(), "conf.d"), "", } diff --git a/cmd/agent/common/path/path_darwin.go b/cmd/agent/common/path/path_darwin.go deleted file mode 100644 index 950a127fce90d..0000000000000 --- a/cmd/agent/common/path/path_darwin.go +++ /dev/null @@ -1,45 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package path - -import ( - "path/filepath" - - "github.com/DataDog/datadog-agent/pkg/util/executable" -) - -const ( - // DefaultConfPath points to the folder containing datadog.yaml - DefaultConfPath = "/opt/datadog-agent/etc" - // DefaultLogFile points to the log file that will be used if not configured - DefaultLogFile = "/opt/datadog-agent/logs/agent.log" - // DefaultDCALogFile points to the log file that will be used if not configured - DefaultDCALogFile = "/opt/datadog-agent/logs/cluster-agent.log" - //DefaultJmxLogFile points to the jmx fetch log file that will be used if not configured - DefaultJmxLogFile = "/opt/datadog-agent/logs/jmxfetch.log" - // DefaultCheckFlareDirectory a flare friendly location for checks to be written - DefaultCheckFlareDirectory = "/opt/datadog-agent/logs/checks/" - // DefaultJMXFlareDirectory a flare friendly location for jmx command logs to be written - DefaultJMXFlareDirectory = "/opt/datadog-agent/logs/jmxinfo/" - //DefaultDogstatsDLogFile points to the dogstatsd stats log file that will be used if not configured - DefaultDogstatsDLogFile = "/opt/datadog-agent/logs/dogstatsd_info/dogstatsd-stats.log" - //DefaultStreamlogsLogFile points to the stream logs log file that will be used if not configured - DefaultStreamlogsLogFile = "/opt/datadog-agent/logs/streamlogs_info/streamlogs.log" -) - -var ( - _here, _ = executable.Folder() - - // PyChecksPath holds the path to the python checks from integrations-core shipped with the agent - PyChecksPath = filepath.Join(_here, "..", "..", "checks.d") - // DistPath holds the path to the folder containing distribution files - distPath = filepath.Join(_here, "dist") -) - -// GetDistPath returns the fully qualified path to the 'dist' directory -func GetDistPath() string { - return distPath -} diff --git a/cmd/agent/common/path/path_freebsd.go b/cmd/agent/common/path/path_freebsd.go deleted file mode 100644 index 3885684a6be81..0000000000000 --- a/cmd/agent/common/path/path_freebsd.go +++ /dev/null @@ -1,45 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package path - -import ( - "path/filepath" - - "github.com/DataDog/datadog-agent/pkg/util/executable" -) - -const ( - // DefaultConfPath points to the folder containing datadog.yaml - DefaultConfPath = "/usr/local/etc/datadog-agent" - // DefaultLogFile points to the log file that will be used if not configured - DefaultLogFile = "/var/log/datadog/agent.log" - // DefaultDCALogFile points to the log file that will be used if not configured - DefaultDCALogFile = "/var/log/datadog/cluster-agent.log" - //DefaultJmxLogFile points to the jmx fetch log file that will be used if not configured - DefaultJmxLogFile = "/var/log/datadog/jmxfetch.log" - // DefaultCheckFlareDirectory a flare friendly location for checks to be written - DefaultCheckFlareDirectory = "/var/log/datadog/checks/" - // DefaultJMXFlareDirectory a flare friendly location for jmx command logs to be written - DefaultJMXFlareDirectory = "/var/log/datadog/jmxinfo/" - //DefaultDogstatsDLogFile points to the dogstatsd stats log file that will be used if not configured - DefaultDogstatsDLogFile = "/var/log/datadog/dogstatsd_info/dogstatsd-stats.log" - //DefaultStreamlogsLogFile points to the stream logs log file that will be used if not configured - DefaultStreamlogsLogFile = "/var/log/datadog/streamlogs_info/streamlogs.log" -) - -var ( - // utility variables - _here, _ = executable.Folder() - // PyChecksPath holds the path to the python checks from integrations-core shipped with the agent - PyChecksPath = filepath.Join(_here, "..", "..", "checks.d") - // DistPath holds the path to the folder containing distribution files - distPath = filepath.Join(_here, "dist") -) - -// GetDistPath returns the fully qualified path to the 'dist' directory -func GetDistPath() string { - return distPath -} diff --git a/cmd/agent/common/path/path_nix.go b/cmd/agent/common/path/path_nix.go deleted file mode 100644 index 479063de13487..0000000000000 --- a/cmd/agent/common/path/path_nix.go +++ /dev/null @@ -1,47 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build netbsd || openbsd || solaris || dragonfly || linux - -package path - -import ( - "path/filepath" - - "github.com/DataDog/datadog-agent/pkg/util/executable" -) - -const ( - // DefaultConfPath points to the folder containing datadog.yaml - DefaultConfPath = "/etc/datadog-agent" - // DefaultLogFile points to the log file that will be used if not configured - DefaultLogFile = "/var/log/datadog/agent.log" - // DefaultDCALogFile points to the log file that will be used if not configured - DefaultDCALogFile = "/var/log/datadog/cluster-agent.log" - // DefaultJmxLogFile points to the jmx fetch log file that will be used if not configured - DefaultJmxLogFile = "/var/log/datadog/jmxfetch.log" - // DefaultCheckFlareDirectory a flare friendly location for checks to be written - DefaultCheckFlareDirectory = "/var/log/datadog/checks/" - // DefaultJMXFlareDirectory a flare friendly location for jmx command logs to be written - DefaultJMXFlareDirectory = "/var/log/datadog/jmxinfo/" - //DefaultDogstatsDLogFile points to the dogstatsd stats log file that will be used if not configured - DefaultDogstatsDLogFile = "/var/log/datadog/dogstatsd_info/dogstatsd-stats.log" - //DefaultStreamlogsLogFile points to the stream logs log file that will be used if not configured - DefaultStreamlogsLogFile = "/var/log/datadog/streamlogs_info/streamlogs.log" -) - -var ( - // utility variables - _here, _ = executable.Folder() - // PyChecksPath holds the path to the python checks from integrations-core shipped with the agent - PyChecksPath = filepath.Join(_here, "..", "..", "checks.d") - // DistPath holds the path to the folder containing distribution files - distPath = filepath.Join(_here, "dist") -) - -// GetDistPath returns the fully qualified path to the 'dist' directory -func GetDistPath() string { - return distPath -} diff --git a/cmd/agent/common/test_helpers.go b/cmd/agent/common/test_helpers.go deleted file mode 100644 index aca2815135a43..0000000000000 --- a/cmd/agent/common/test_helpers.go +++ /dev/null @@ -1,55 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build test - -package common - -import ( - "errors" - "fmt" - "io/fs" - "runtime" - "strings" - - "github.com/DataDog/datadog-agent/cmd/agent/common/path" - "github.com/DataDog/datadog-agent/comp/core/secrets" - "github.com/DataDog/datadog-agent/pkg/config/model" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/util/optional" -) - -// SetupConfigForTest fires up the configuration system and returns warnings if any. -func SetupConfigForTest(confFilePath string) (*model.Warnings, error) { - cfg := pkgconfigsetup.Datadog() - origin := "datadog.yaml" - // set the paths where a config file is expected - if len(confFilePath) != 0 { - // if the configuration file path was supplied on the command line, - // add that first so it's first in line - cfg.AddConfigPath(confFilePath) - // If they set a config file directly, let's try to honor that - if strings.HasSuffix(confFilePath, ".yaml") { - cfg.SetConfigFile(confFilePath) - } - } - cfg.AddConfigPath(path.DefaultConfPath) - // load the configuration - warnings, err := pkgconfigsetup.LoadDatadogCustom(cfg, origin, optional.NewNoneOption[secrets.Component](), nil) - if err != nil { - // special-case permission-denied with a clearer error message - if errors.Is(err, fs.ErrPermission) { - if runtime.GOOS == "windows" { - err = fmt.Errorf(`cannot access the Datadog config file (%w); try running the command in an Administrator shell"`, err) - } else { - err = fmt.Errorf("cannot access the Datadog config file (%w); try running the command under the same user as the Datadog Agent", err) - } - } else { - err = fmt.Errorf("unable to load Datadog config file: %w", err) - } - return warnings, err - } - return warnings, nil -} diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index c39613e753945..8c159e673cacd 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -23,8 +23,6 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/command" "github.com/DataDog/datadog-agent/cmd/agent/common" "github.com/DataDog/datadog-agent/cmd/agent/subcommands/streamlogs" - - commonpath "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl" authtokenimpl "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/collector/collector" @@ -53,6 +51,7 @@ import ( pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/process/net" "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/input" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -92,7 +91,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { cliParams.args = args config := config.NewAgentParams(globalParams.ConfFilePath, config.WithSecurityAgentConfigFilePaths([]string{ - path.Join(commonpath.DefaultConfPath, "security-agent.yaml"), + path.Join(defaultpaths.ConfPath, "security-agent.yaml"), }), config.WithConfigLoadSecurityAgent(true), config.WithIgnoreErrors(true), @@ -109,12 +108,12 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { LogParams: log.ForOneShot(command.LoggerName, "off", false), }), flare.Module(flare.NewLocalParams( - commonpath.GetDistPath(), - commonpath.PyChecksPath, - commonpath.DefaultLogFile, - commonpath.DefaultJmxLogFile, - commonpath.DefaultDogstatsDLogFile, - commonpath.DefaultStreamlogsLogFile, + defaultpaths.GetDistPath(), + defaultpaths.PyChecksPath, + defaultpaths.LogFile, + defaultpaths.JmxLogFile, + defaultpaths.DogstatsDLogFile, + defaultpaths.StreamlogsLogFile, )), // workloadmeta setup wmcatalog.GetCatalog(), @@ -285,7 +284,7 @@ func makeFlare(flareComp flare.Component, ) streamLogParams := streamlogs.CliParams{ - FilePath: commonpath.DefaultStreamlogsLogFile, + FilePath: defaultpaths.StreamlogsLogFile, Duration: cliParams.withStreamLogs, Quiet: true, } diff --git a/cmd/agent/subcommands/flare/command_other_test.go b/cmd/agent/subcommands/flare/command_other_test.go new file mode 100644 index 0000000000000..6730af25dcece --- /dev/null +++ b/cmd/agent/subcommands/flare/command_other_test.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build !windows + +// Package flare implements 'agent flare'. +package flare + +import ( + "net/http" + "net/http/httptest" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/config/model" +) + +// NewSystemProbeTestServer starts a new mock server to handle System Probe requests. +func NewSystemProbeTestServer(_ http.Handler) (*httptest.Server, error) { + // Linux still uses a port-based system-probe, it does not need a dedicated system probe server + // for the tests. + return nil, nil +} + +// InjectConnectionFailures injects a failure in TestReadProfileDataErrors. +func InjectConnectionFailures(_ model.Config, _ model.Config) { +} + +// CheckExpectedConnectionFailures checks the expected errors after simulated +// connection failures. +func CheckExpectedConnectionFailures(c *commandTestSuite, err error) { + // System probe by default is disabled and no connection is attempted for it in the test. + require.Regexp(c.T(), "^4 errors occurred:\n", err.Error()) +} diff --git a/cmd/agent/subcommands/flare/command_test.go b/cmd/agent/subcommands/flare/command_test.go index 5dbeeb2c68376..b555c751f58b5 100644 --- a/cmd/agent/subcommands/flare/command_test.go +++ b/cmd/agent/subcommands/flare/command_test.go @@ -31,24 +31,38 @@ type commandTestSuite struct { sysprobeSocketPath string tcpServer *httptest.Server unixServer *httptest.Server + systemProbeServer *httptest.Server } func (c *commandTestSuite) SetupSuite() { t := c.T() c.sysprobeSocketPath = path.Join(t.TempDir(), "sysprobe.sock") - c.tcpServer, c.unixServer = c.getPprofTestServer() } -func (c *commandTestSuite) TearDownSuite() { - c.tcpServer.Close() - if c.unixServer != nil { - c.unixServer.Close() - } +// startTestServers starts test servers from a clean state to ensure no cache responses are used. +// This should be called by each test that requires them. +func (c *commandTestSuite) startTestServers() { + t := c.T() + c.tcpServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() + + t.Cleanup(func() { + if c.tcpServer != nil { + c.tcpServer.Close() + c.tcpServer = nil + } + if c.unixServer != nil { + c.unixServer.Close() + c.unixServer = nil + } + if c.systemProbeServer != nil { + c.systemProbeServer.Close() + c.systemProbeServer = nil + } + }) } -func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server) { - t := c.T() - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func newMockHandler() http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/debug/pprof/heap": w.Write([]byte("heap_profile")) @@ -67,17 +81,28 @@ func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, uni w.WriteHeader(500) } }) +} +func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { + var err error + t := c.T() + + handler := newMockHandler() tcpServer = httptest.NewServer(handler) if runtime.GOOS == "linux" { unixServer = httptest.NewUnstartedServer(handler) - var err error unixServer.Listener, err = net.Listen("unix", c.sysprobeSocketPath) require.NoError(t, err, "could not create listener for unix socket on %s", c.sysprobeSocketPath) unixServer.Start() } - return tcpServer, unixServer + sysProbeServer, err = NewSystemProbeTestServer(handler) + require.NoError(c.T(), err, "could not restart system probe server") + if sysProbeServer != nil { + sysProbeServer.Start() + } + + return tcpServer, unixServer, sysProbeServer } func TestCommandTestSuite(t *testing.T) { @@ -86,6 +111,8 @@ func TestCommandTestSuite(t *testing.T) { func (c *commandTestSuite) TestReadProfileData() { t := c.T() + c.startTestServers() + u, err := url.Parse(c.tcpServer.URL) require.NoError(t, err) port := u.Port() @@ -154,6 +181,8 @@ func (c *commandTestSuite) TestReadProfileData() { func (c *commandTestSuite) TestReadProfileDataNoTraceAgent() { t := c.T() + c.startTestServers() + u, err := url.Parse(c.tcpServer.URL) require.NoError(t, err) port := u.Port() @@ -217,6 +246,8 @@ func (c *commandTestSuite) TestReadProfileDataNoTraceAgent() { func (c *commandTestSuite) TestReadProfileDataErrors() { t := c.T() + c.startTestServers() + mockConfig := configmock.New(t) // setting Core Agent Expvar port to 0 to ensure failing on fetch (using the default value can lead to // successful request when running next to an Agent) @@ -226,9 +257,13 @@ func (c *commandTestSuite) TestReadProfileDataErrors() { mockConfig.SetWithoutSource("process_config.enabled", true) mockConfig.SetWithoutSource("process_config.expvar_port", 0) + mockSysProbeConfig := configmock.NewSystemProbe(t) + InjectConnectionFailures(mockSysProbeConfig, mockConfig) + data, err := readProfileData(10) + require.Error(t, err) - require.Regexp(t, "^4 errors occurred:\n", err.Error()) + CheckExpectedConnectionFailures(c, err) require.Len(t, data, 0) } diff --git a/cmd/agent/subcommands/flare/command_windows_test.go b/cmd/agent/subcommands/flare/command_windows_test.go new file mode 100644 index 0000000000000..9780e66474033 --- /dev/null +++ b/cmd/agent/subcommands/flare/command_windows_test.go @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build windows + +// Package flare implements 'agent flare'. +package flare + +import ( + "net/http" + "net/http/httptest" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/config/model" + processNet "github.com/DataDog/datadog-agent/pkg/process/net" +) + +const ( + // SystemProbeTestPipeName is the test named pipe for system-probe + systemProbeTestPipeName = `\\.\pipe\dd_system_probe_flare_test` + + // systemProbeTestPipeSecurityDescriptor has a DACL that allows Everyone access for these tests. + systemProbeTestPipeSecurityDescriptor = "D:PAI(A;;FA;;;WD)" +) + +// NewSystemProbeTestServer starts a new mock server to handle System Probe requests. +func NewSystemProbeTestServer(handler http.Handler) (*httptest.Server, error) { + server := httptest.NewUnstartedServer(handler) + + // Override the named pipe path for tests to avoid conflicts with the locally installed Datadog agent. + processNet.OverrideSystemProbeNamedPipeConfig( + systemProbeTestPipeName, + systemProbeTestPipeSecurityDescriptor) + + conn, err := processNet.NewSystemProbeListener("") + if err != nil { + return nil, err + } + + server.Listener = conn.GetListener() + return server, nil +} + +// InjectConnectionFailures injects a failure in TestReadProfileDataErrors. +func InjectConnectionFailures(mockSysProbeConfig model.Config, mockConfig model.Config) { + // Explicitly enabled system probe to exercise connections to it. + mockSysProbeConfig.SetWithoutSource("system_probe_config.enabled", true) + + // Exercise a connection failure for a Windows system probe named pipe client by + // making them use a bad path. + // The system probe http server must be setup before this override. + processNet.OverrideSystemProbeNamedPipeConfig( + `\\.\pipe\dd_system_probe_test_bad`, + systemProbeTestPipeSecurityDescriptor) + + // The security-agent connection is expected to fail too in this test, but + // by enabling system probe, a port will be provided to it (security agent). + // Here we make sure the security agent port is a bad one. + mockConfig.SetWithoutSource("security_agent.expvar_port", 0) +} + +// CheckExpectedConnectionFailures checks the expected errors after simulated +// connection failures. +func CheckExpectedConnectionFailures(c *commandTestSuite, err error) { + // In Windows, this test explicitly simulates a system probe connection failure. + // We expect the standard socket errors (4) and a named pipe failure for system probe. + require.Regexp(c.T(), "^5 errors occurred:\n", err.Error()) +} diff --git a/cmd/agent/subcommands/jmx/command.go b/cmd/agent/subcommands/jmx/command.go index b827558dbdb73..b56915eb1b406 100644 --- a/cmd/agent/subcommands/jmx/command.go +++ b/cmd/agent/subcommands/jmx/command.go @@ -20,7 +20,6 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/command" "github.com/DataDog/datadog-agent/cmd/agent/common" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/agent" "github.com/DataDog/datadog-agent/comp/agent/jmxlogger" "github.com/DataDog/datadog-agent/comp/agent/jmxlogger/jmxloggerimpl" @@ -59,6 +58,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/cli/standalone" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -107,7 +107,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { if cliParams.saveFlare { // Windows cannot accept ":" in file names filenameSafeTimeStamp := strings.ReplaceAll(time.Now().UTC().Format(time.RFC3339), ":", "-") - cliParams.logFile = filepath.Join(path.DefaultJMXFlareDirectory, "jmx_"+cliParams.command+"_"+filenameSafeTimeStamp+".log") + cliParams.logFile = filepath.Join(defaultpaths.JMXFlareDirectory, "jmx_"+cliParams.command+"_"+filenameSafeTimeStamp+".log") cliParams.jmxLogLevel = "debug" } diff --git a/cmd/agent/subcommands/jmx/command_test.go b/cmd/agent/subcommands/jmx/command_test.go index 8bc173ac46037..bd162de9cc1bd 100644 --- a/cmd/agent/subcommands/jmx/command_test.go +++ b/cmd/agent/subcommands/jmx/command_test.go @@ -14,9 +14,9 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/cmd/agent/command" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/secrets" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -64,7 +64,7 @@ func TestCollectCommand(t *testing.T) { require.Equal(t, "collect", cliParams.command) require.Equal(t, "debug", cliParams.jmxLogLevel) // overrides --log-level require.Equal(t, "debug", coreParams.LogLevelFn(nil)) // overrides --log-level - require.True(t, strings.HasPrefix(cliParams.logFile, path.DefaultJMXFlareDirectory)) + require.True(t, strings.HasPrefix(cliParams.logFile, defaultpaths.JMXFlareDirectory)) require.Equal(t, cliParams.logFile, coreParams.LogFileFn(nil)) require.Equal(t, "CORE", coreParams.LoggerName()) require.Equal(t, true, secretParams.Enabled) diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 34d8bc3e2225e..c1cf43618fe79 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -25,7 +25,6 @@ import ( "github.com/DataDog/datadog-agent/cmd/agent/command" "github.com/DataDog/datadog-agent/cmd/agent/common" "github.com/DataDog/datadog-agent/cmd/agent/common/misconfig" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/cmd/agent/common/signals" "github.com/DataDog/datadog-agent/cmd/agent/subcommands/run/internal/clcrunnerapi" internalsettings "github.com/DataDog/datadog-agent/cmd/agent/subcommands/run/internal/settings" @@ -146,8 +145,10 @@ import ( pkgTelemetry "github.com/DataDog/datadog-agent/pkg/telemetry" "github.com/DataDog/datadog-agent/pkg/util" pkgcommon "github.com/DataDog/datadog-agent/pkg/util/common" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/flavor" "github.com/DataDog/datadog-agent/pkg/util/fxutil" + "github.com/DataDog/datadog-agent/pkg/util/fxutil/logging" "github.com/DataDog/datadog-agent/pkg/util/hostname" "github.com/DataDog/datadog-agent/pkg/util/installinfo" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/leaderelection" @@ -182,9 +183,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { ConfigParams: config.NewAgentParams(globalParams.ConfFilePath, config.WithExtraConfFiles(cliParams.ExtraConfFilePath), config.WithFleetPoliciesDirPath(cliParams.FleetPoliciesDirPath)), SecretParams: secrets.NewEnabledParams(), SysprobeConfigParams: sysprobeconfigimpl.NewParams(sysprobeconfigimpl.WithSysProbeConfFilePath(globalParams.SysProbeConfFilePath), sysprobeconfigimpl.WithFleetPoliciesDirPath(cliParams.FleetPoliciesDirPath)), - LogParams: log.ForDaemon(command.LoggerName, "log_file", path.DefaultLogFile), + LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.LogFile), }), fx.Supply(pidimpl.NewParams(cliParams.pidfilePath)), + logging.EnableFxLoggingOnDebug[log.Component](), getSharedFxOption(), getPlatformModules(), ) @@ -329,12 +331,12 @@ func run(log log.Component, func getSharedFxOption() fx.Option { return fx.Options( flare.Module(flare.NewParams( - path.GetDistPath(), - path.PyChecksPath, - path.DefaultLogFile, - path.DefaultJmxLogFile, - path.DefaultDogstatsDLogFile, - path.DefaultStreamlogsLogFile, + defaultpaths.GetDistPath(), + defaultpaths.PyChecksPath, + defaultpaths.LogFile, + defaultpaths.JmxLogFile, + defaultpaths.DogstatsDLogFile, + defaultpaths.StreamlogsLogFile, )), core.Bundle(), lsof.Module(), @@ -376,7 +378,7 @@ func getSharedFxOption() fx.Option { apiimpl.Module(), commonendpoints.Module(), compressionimpl.Module(), - demultiplexerimpl.Module(), + demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams(demultiplexerimpl.WithDogstatsdNoAggregationPipelineConfig())), demultiplexerendpointfx.Module(), dogstatsd.Bundle(dogstatsdServer.Params{Serverless: false}), fx.Provide(func(logsagent optional.Option[logsAgent.Component]) optional.Option[logsagentpipeline.Component] { @@ -416,15 +418,7 @@ func getSharedFxOption() fx.Option { logs.Bundle(), langDetectionClimpl.Module(), metadata.Bundle(), - // injecting the aggregator demultiplexer to FX until we migrate it to a proper component. This allows - // other already migrated components to request it. - fx.Provide(func(config config.Component) demultiplexerimpl.Params { - params := demultiplexerimpl.NewDefaultParams() - params.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline") - return params - }), - orchestratorForwarderImpl.Module(), - fx.Supply(orchestratorForwarderImpl.NewDefaultParams()), + orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDefaultParams()), eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()), eventplatformreceiverimpl.Module(), diff --git a/cmd/agent/subcommands/run/command_windows.go b/cmd/agent/subcommands/run/command_windows.go index fc3a05bec3adf..e4bb454a1f182 100644 --- a/cmd/agent/subcommands/run/command_windows.go +++ b/cmd/agent/subcommands/run/command_windows.go @@ -28,7 +28,6 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/cmd/agent/command" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/cmd/agent/common/signals" // checks implemented as components @@ -74,6 +73,7 @@ import ( processAgent "github.com/DataDog/datadog-agent/comp/process/agent" "github.com/DataDog/datadog-agent/comp/remote-config/rcclient" "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" // runtime init routines @@ -186,7 +186,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error ConfigParams: config.NewAgentParams(""), SecretParams: secrets.NewEnabledParams(), SysprobeConfigParams: sysprobeconfigimpl.NewParams(), - LogParams: log.ForDaemon(command.LoggerName, "log_file", path.DefaultLogFile), + LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.LogFile), }), getSharedFxOption(), getPlatformModules(), diff --git a/cmd/agent/subcommands/snmp/command.go b/cmd/agent/subcommands/snmp/command.go index b19402aec82f0..1b5dbf6beedec 100644 --- a/cmd/agent/subcommands/snmp/command.go +++ b/cmd/agent/subcommands/snmp/command.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/DataDog/datadog-agent/comp/serializer/compression/compressionimpl" "net" "os" "strconv" @@ -175,8 +176,9 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { snmpCmd.AddCommand(snmpWalkCmd) snmpScanCmd := &cobra.Command{ - Use: "scan [:port]", - Short: "Scan a device for the profile editor.", + Hidden: true, + Use: "scan [:port]", + Short: "Scan a device for the profile editor.", Long: `Walk the SNMP tree for a device, collecting available OIDs. Flags that aren't specified will be pulled from the agent SNMP config if possible.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -184,20 +186,17 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { err := fxutil.OneShot(scanDevice, fx.Supply(connParams, globalParams, cmd), fx.Provide(func() argsType { return args }), + compressionimpl.Module(), fx.Supply(core.BundleParams{ ConfigParams: config.NewAgentParams(globalParams.ConfFilePath, config.WithExtraConfFiles(globalParams.ExtraConfFilePath), config.WithFleetPoliciesDirPath(globalParams.FleetPoliciesDirPath)), SecretParams: secrets.NewEnabledParams(), LogParams: log.ForOneShot(command.LoggerName, "off", true)}), core.Bundle(), - aggregator.Bundle(), + aggregator.Bundle(demultiplexerimpl.NewDefaultParams()), forwarder.Bundle(defaultforwarder.NewParams()), eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()), eventplatformreceiverimpl.Module(), - orchestratorimpl.Module(), - fx.Provide( - orchestratorimpl.NewDefaultParams, - demultiplexerimpl.NewDefaultParams, - ), + orchestratorimpl.Module(orchestratorimpl.NewDefaultParams()), ) if err != nil { var ue configErr @@ -230,7 +229,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { snmpScanCmd.Flags().BoolVar(&connParams.UseUnconnectedUDPSocket, "use-unconnected-udp-socket", defaultUseUnconnectedUDPSocket, "If specified, changes net connection to be unconnected UDP socket") // This command does nothing until the backend supports it, so it isn't enabled yet. - // snmpCmd.AddCommand(snmpScanCmd) + snmpCmd.AddCommand(snmpScanCmd) return []*cobra.Command{snmpCmd} } diff --git a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go index 59415f5ec8146..aac5d3639a948 100644 --- a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go +++ b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra" "github.com/DataDog/datadog-agent/cmd/agent/common" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/cmd/cluster-agent-cloudfoundry/command" "github.com/DataDog/datadog-agent/cmd/cluster-agent/api" dcav1 "github.com/DataDog/datadog-agent/cmd/cluster-agent/api/v1" @@ -59,6 +58,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/cloudproviders/cloudfoundry" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/hostname" pkglog "github.com/DataDog/datadog-agent/pkg/util/log" @@ -80,17 +80,16 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Supply(core.BundleParams{ ConfigParams: config.NewClusterAgentParams(globalParams.ConfFilePath), SecretParams: secrets.NewEnabledParams(), - LogParams: log.ForDaemon(command.LoggerName, "log_file", path.DefaultDCALogFile), + LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.DCALogFile), }), core.Bundle(), forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithResolvers())), compressionimpl.Module(), - demultiplexerimpl.Module(), - orchestratorForwarderImpl.Module(), - fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), + demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()), + orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDisabledParams()), eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()), eventplatformreceiverimpl.Module(), - fx.Supply(demultiplexerimpl.NewDefaultParams()), + // setup workloadmeta wmcatalog.GetCatalog(), workloadmetafx.Module(workloadmeta.Params{ diff --git a/cmd/cluster-agent/admission/server.go b/cmd/cluster-agent/admission/server.go index bff5012e1c4c1..5af71864f65c1 100644 --- a/cmd/cluster-agent/admission/server.go +++ b/cmd/cluster-agent/admission/server.go @@ -18,30 +18,28 @@ import ( "net/http" "time" - authenticationv1 "k8s.io/api/authentication/v1" - "github.com/cihub/seelog" + admiv1 "k8s.io/api/admission/v1" + admiv1beta1 "k8s.io/api/admission/v1beta1" + authenticationv1 "k8s.io/api/authentication/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + admicommon "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/common" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/metrics" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/certificate" "github.com/DataDog/datadog-agent/pkg/util/log" pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup" - - admiv1 "k8s.io/api/admission/v1" - admiv1beta1 "k8s.io/api/admission/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" ) const jsonContentType = "application/json" -// MutateRequest contains the information of a mutation request -type MutateRequest struct { +// Request contains the information of an admission request +type Request struct { // Raw is the raw request object Raw []byte // Name is the name of the object @@ -56,8 +54,9 @@ type MutateRequest struct { APIClient kubernetes.Interface } -// WebhookFunc is the function that runs the webhook logic -type WebhookFunc func(request *MutateRequest) ([]byte, error) +// WebhookFunc is the function that runs the webhook logic. +// We always return an `admissionv1.AdmissionResponse` as it will be converted to the appropriate version if needed. +type WebhookFunc func(request *Request) *admiv1.AdmissionResponse // Server TODO type Server struct { @@ -94,9 +93,9 @@ func (s *Server) initDecoder() { // Register adds an admission webhook handler. // Register must be called to register the desired webhook handlers before calling Run. -func (s *Server) Register(uri string, webhookName string, f WebhookFunc, dc dynamic.Interface, apiClient kubernetes.Interface) { +func (s *Server) Register(uri string, webhookName string, webhookType admicommon.WebhookType, f WebhookFunc, dc dynamic.Interface, apiClient kubernetes.Interface) { s.mux.HandleFunc(uri, func(w http.ResponseWriter, r *http.Request) { - s.mutateHandler(w, r, webhookName, f, dc, apiClient) + s.handle(w, r, webhookName, webhookType, f, dc, apiClient) }) } @@ -136,16 +135,21 @@ func (s *Server) Run(mainCtx context.Context, client kubernetes.Interface) error return server.Shutdown(shutdownCtx) } -// mutateHandler contains the main logic responsible for handling mutation requests. +// handle contains the main logic responsible for handling admission requests. // It supports both v1 and v1beta1 requests. -func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookName string, mutateFunc WebhookFunc, dc dynamic.Interface, apiClient kubernetes.Interface) { - metrics.WebhooksReceived.Inc(webhookName) +func (s *Server) handle(w http.ResponseWriter, r *http.Request, webhookName string, webhookType admicommon.WebhookType, webhookFunc WebhookFunc, dc dynamic.Interface, apiClient kubernetes.Interface) { + // Increment the metrics for the received webhook. + // We send the webhook name twice to keep the backward compatibility with `mutation_type` tag. + metrics.WebhooksReceived.Inc(webhookName, webhookName, webhookType.String()) + // Measure the time it takes to process the request. start := time.Now() defer func() { - metrics.WebhooksResponseDuration.Observe(time.Since(start).Seconds(), webhookName) + // We send the webhook name twice to keep the backward compatibility with `mutation_type` tag. + metrics.WebhooksResponseDuration.Observe(time.Since(start).Seconds(), webhookName, webhookName, webhookType.String()) }() + // Validate admission request. if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) log.Warnf("Invalid method %s, only POST requests are allowed", r.Method) @@ -166,6 +170,7 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa return } + // Deserialize admission request. obj, gvk, err := s.decoder.Decode(body, nil, nil) if err != nil { w.WriteHeader(http.StatusBadRequest) @@ -173,6 +178,7 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa return } + // Handle the request based on `GroupVersionKind`. var response runtime.Object switch *gvk { case admiv1.SchemeGroupVersion.WithKind("AdmissionReview"): @@ -180,9 +186,10 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa if !ok { log.Errorf("Expected v1.AdmissionReview, got type %T", obj) } - admissionReviewResp := &admiv1.AdmissionReview{} - admissionReviewResp.SetGroupVersionKind(*gvk) - mutateRequest := MutateRequest{ + + admissionReview := &admiv1.AdmissionReview{} + admissionReview.SetGroupVersionKind(*gvk) + admissionRequest := Request{ Raw: admissionReviewReq.Request.Object.Raw, Name: admissionReviewReq.Request.Name, Namespace: admissionReviewReq.Request.Namespace, @@ -190,18 +197,21 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa DynamicClient: dc, APIClient: apiClient, } - jsonPatch, err := mutateFunc(&mutateRequest) - admissionReviewResp.Response = mutationResponse(jsonPatch, err) - admissionReviewResp.Response.UID = admissionReviewReq.Request.UID - response = admissionReviewResp + + // Generate admission response + admissionResponse := webhookFunc(&admissionRequest) + admissionReview.Response = admissionResponse + admissionReview.Response.UID = admissionReviewReq.Request.UID + response = admissionReview case admiv1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): admissionReviewReq, ok := obj.(*admiv1beta1.AdmissionReview) if !ok { log.Errorf("Expected v1beta1.AdmissionReview, got type %T", obj) } - admissionReviewResp := &admiv1beta1.AdmissionReview{} - admissionReviewResp.SetGroupVersionKind(*gvk) - mutateRequest := MutateRequest{ + + admissionReview := &admiv1beta1.AdmissionReview{} + admissionReview.SetGroupVersionKind(*gvk) + admissionRequest := Request{ Raw: admissionReviewReq.Request.Object.Raw, Name: admissionReviewReq.Request.Name, Namespace: admissionReviewReq.Request.Namespace, @@ -209,10 +219,12 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa DynamicClient: dc, APIClient: apiClient, } - jsonPatch, err := mutateFunc(&mutateRequest) - admissionReviewResp.Response = responseV1ToV1beta1(mutationResponse(jsonPatch, err)) - admissionReviewResp.Response.UID = admissionReviewReq.Request.UID - response = admissionReviewResp + + // Generate admission response + admissionResponse := webhookFunc(&admissionRequest) + admissionReview.Response = responseV1ToV1beta1(admissionResponse) + admissionReview.Response.UID = admissionReviewReq.Request.UID + response = admissionReview default: log.Errorf("Group version kind %v is not supported", gvk) w.WriteHeader(http.StatusBadRequest) @@ -228,29 +240,6 @@ func (s *Server) mutateHandler(w http.ResponseWriter, r *http.Request, webhookNa } } -// mutationResponse returns the adequate v1.AdmissionResponse based on the mutation result. -func mutationResponse(jsonPatch []byte, err error) *admiv1.AdmissionResponse { - if err != nil { - log.Warnf("Failed to mutate: %v", err) - - return &admiv1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - Allowed: true, - } - - } - - patchType := admiv1.PatchTypeJSONPatch - - return &admiv1.AdmissionResponse{ - Patch: jsonPatch, - PatchType: &patchType, - Allowed: true, - } -} - // responseV1ToV1beta1 converts a v1.AdmissionResponse into a v1beta1.AdmissionResponse. func responseV1ToV1beta1(resp *admiv1.AdmissionResponse) *admiv1beta1.AdmissionResponse { var patchType *admiv1beta1.PatchType diff --git a/cmd/cluster-agent/api/agent/agent.go b/cmd/cluster-agent/api/agent/agent.go index 86a536d77cb20..d5d3ded9ebdb3 100644 --- a/cmd/cluster-agent/api/agent/agent.go +++ b/cmd/cluster-agent/api/agent/agent.go @@ -15,7 +15,6 @@ import ( "github.com/gorilla/mux" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/cmd/agent/common/signals" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" "github.com/DataDog/datadog-agent/comp/core/settings" @@ -25,6 +24,7 @@ import ( pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/flare" "github.com/DataDog/datadog-agent/pkg/status/health" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/hostname" httputils "github.com/DataDog/datadog-agent/pkg/util/http" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -146,9 +146,9 @@ func makeFlare(w http.ResponseWriter, r *http.Request, statusComponent status.Co logFile := pkgconfigsetup.Datadog().GetString("log_file") if logFile == "" { - logFile = path.DefaultDCALogFile + logFile = defaultpaths.DCALogFile } - filePath, err := flare.CreateDCAArchive(false, path.GetDistPath(), logFile, profile, statusComponent) + filePath, err := flare.CreateDCAArchive(false, defaultpaths.GetDistPath(), logFile, profile, statusComponent) if err != nil || filePath == "" { if err != nil { log.Errorf("The flare failed to be created: %s", err) diff --git a/cmd/cluster-agent/subcommands/start/command.go b/cmd/cluster-agent/subcommands/start/command.go index 786641cc7c263..37b1305a9ec9d 100644 --- a/cmd/cluster-agent/subcommands/start/command.go +++ b/cmd/cluster-agent/subcommands/start/command.go @@ -10,6 +10,7 @@ package start import ( "context" + "errors" "fmt" "net/http" "os" @@ -20,11 +21,11 @@ import ( "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/controllers" "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/cmd/agent/common" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" admissioncmd "github.com/DataDog/datadog-agent/cmd/cluster-agent/admission" "github.com/DataDog/datadog-agent/cmd/cluster-agent/api" dcav1 "github.com/DataDog/datadog-agent/cmd/cluster-agent/api/v1" @@ -131,17 +132,15 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { fx.Supply(core.BundleParams{ ConfigParams: config.NewClusterAgentParams(globalParams.ConfFilePath), SecretParams: secrets.NewEnabledParams(), - LogParams: log.ForDaemon(command.LoggerName, "log_file", path.DefaultDCALogFile), + LogParams: log.ForDaemon(command.LoggerName, "log_file", defaultpaths.DCALogFile), }), core.Bundle(), forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithResolvers(), defaultforwarder.WithDisableAPIKeyChecking())), compressionimpl.Module(), - demultiplexerimpl.Module(), - orchestratorForwarderImpl.Module(), - fx.Supply(orchestratorForwarderImpl.NewDefaultParams()), + demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()), + orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDefaultParams()), eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()), eventplatformreceiverimpl.Module(), - fx.Supply(demultiplexerimpl.NewDefaultParams()), // setup workloadmeta wmcatalog.GetCatalog(), workloadmetafx.Module(workloadmeta.Params{ @@ -219,8 +218,10 @@ func start(log log.Component, logReceiver optional.Option[integrations.Component], _ healthprobe.Component, settings settings.Component, + datadogConfig config.Component, ) error { stopCh := make(chan struct{}) + validatingStopCh := make(chan struct{}) mainCtx, mainCtxCancel := context.WithCancel(context.Background()) defer mainCtxCancel() @@ -371,7 +372,7 @@ func start(log log.Component, common.LoadComponents(secretResolver, wmeta, ac, config.GetString("confd_path")) // Set up check collector - registerChecks() + registerChecks(wmeta, config) ac.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer, logReceiver), true) // start the autoconfig, this will immediately run any configured check @@ -464,21 +465,30 @@ func start(log log.Component, IsLeaderFunc: le.IsLeader, LeaderSubscribeFunc: le.Subscribe, SecretInformers: apiCl.CertificateSecretInformerFactory, - WebhookInformers: apiCl.WebhookConfigInformerFactory, + ValidatingInformers: apiCl.WebhookConfigInformerFactory, + MutatingInformers: apiCl.WebhookConfigInformerFactory, Client: apiCl.Cl, StopCh: stopCh, + ValidatingStopCh: validatingStopCh, } - webhooks, err := admissionpkg.StartControllers(admissionCtx, wmeta, pa) - if err != nil { + webhooks, err := admissionpkg.StartControllers(admissionCtx, wmeta, pa, datadogConfig) + // Ignore the error if it's related to the validatingwebhookconfigurations. + var syncInformerError *apiserver.SyncInformersError + if err != nil && !(errors.As(err, &syncInformerError) && syncInformerError.Name == apiserver.ValidatingWebhooksInformer) { pkglog.Errorf("Could not start admission controller: %v", err) } else { + if err != nil { + pkglog.Warnf("Admission controller started with errors: %v", err) + pkglog.Debugf("Closing ValidatingWebhooksInformer channel") + close(validatingStopCh) + } // Webhook and secret controllers are started successfully - // Setup the k8s admission webhook server + // Set up the k8s admission webhook server server := admissioncmd.NewServer() for _, webhookConf := range webhooks { - server.Register(webhookConf.Endpoint(), webhookConf.Name(), webhookConf.MutateFunc(), apiCl.DynamicCl, apiCl.Cl) + server.Register(webhookConf.Endpoint(), webhookConf.Name(), webhookConf.WebhookType(), webhookConf.WebhookFunc(), apiCl.DynamicCl, apiCl.Cl) } // Start the k8s admission webhook server @@ -518,6 +528,9 @@ func start(log log.Component, wg.Wait() close(stopCh) + if validatingStopCh != nil { + close(validatingStopCh) + } if err := metricsServer.Shutdown(context.Background()); err != nil { pkglog.Errorf("Error shutdowning metrics server on port %d: %v", metricsPort, err) @@ -564,7 +577,7 @@ func initializeRemoteConfigClient(rcService rccomp.Component, config config.Comp return rcClient, nil } -func registerChecks() { +func registerChecks(wlm workloadmeta.Component, cfg config.Component) { // Required checks corecheckLoader.RegisterCheck(cpu.CheckName, cpu.Factory()) corecheckLoader.RegisterCheck(memory.CheckName, memory.Factory()) @@ -577,6 +590,6 @@ func registerChecks() { corecheckLoader.RegisterCheck(ksm.CheckName, ksm.Factory()) corecheckLoader.RegisterCheck(helm.CheckName, helm.Factory()) corecheckLoader.RegisterCheck(disk.CheckName, disk.Factory()) - corecheckLoader.RegisterCheck(orchestrator.CheckName, orchestrator.Factory()) + corecheckLoader.RegisterCheck(orchestrator.CheckName, orchestrator.Factory(wlm, cfg)) corecheckLoader.RegisterCheck(winproc.CheckName, winproc.Factory()) } diff --git a/cmd/cws-instrumentation/subcommands/tracecmd/trace.go b/cmd/cws-instrumentation/subcommands/tracecmd/trace.go index 14dddd07ed715..c1b393d8721f1 100644 --- a/cmd/cws-instrumentation/subcommands/tracecmd/trace.go +++ b/cmd/cws-instrumentation/subcommands/tracecmd/trace.go @@ -204,7 +204,12 @@ func Command() []*cobra.Command { return nil } - return ptracer.Wrap(args, os.Environ(), params.ProbeAddr, opts) + exitCode, err := ptracer.Wrap(args, os.Environ(), params.ProbeAddr, opts) + if err != nil { + return err + } + os.Exit(exitCode) + return err // fake return }, } diff --git a/cmd/dogstatsd/subcommands/start/command.go b/cmd/dogstatsd/subcommands/start/command.go index bbfd9bb5b0f74..0e1193634a0b0 100644 --- a/cmd/dogstatsd/subcommands/start/command.go +++ b/cmd/dogstatsd/subcommands/start/command.go @@ -149,10 +149,12 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil }), compressionimpl.Module(), - demultiplexerimpl.Module(), + demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams( + demultiplexerimpl.WithContinueOnMissingHostname(), + demultiplexerimpl.WithDogstatsdNoAggregationPipelineConfig(), + )), secretsimpl.Module(), - orchestratorForwarderImpl.Module(), - fx.Supply(orchestratorForwarderImpl.NewDisabledParams()), + orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDisabledParams()), eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()), eventplatformreceiverimpl.Module(), hostnameimpl.Module(), @@ -162,12 +164,6 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil fx.Provide(func(demuxInstance demultiplexer.Component) serializer.MetricSerializer { return demuxInstance.Serializer() }), - fx.Provide(func(config config.Component) demultiplexerimpl.Params { - params := demultiplexerimpl.NewDefaultParams() - params.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline") - params.ContinueOnMissingHostname = true - return params - }), fx.Supply(resourcesimpl.Disabled()), metadatarunnerimpl.Module(), resourcesimpl.Module(), diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 51bd2ce7bec57..15fcd2a8eb91c 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2023-present Datadog, Inc. +//go:build !bootstrapper + // Package main implements 'installer'. package main diff --git a/cmd/installer/main_bootstrapper.go b/cmd/installer/main_bootstrapper.go new file mode 100644 index 0000000000000..25f95f8ef281f --- /dev/null +++ b/cmd/installer/main_bootstrapper.go @@ -0,0 +1,28 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +//go:build bootstrapper + +// Package main implements 'installer'. +package main + +import ( + "fmt" + "os" + + "github.com/DataDog/datadog-agent/cmd/installer/subcommands/installer" + "github.com/DataDog/datadog-agent/cmd/installer/user" + "github.com/DataDog/datadog-agent/cmd/internal/runcmd" +) + +func main() { + if !user.IsRoot() { + fmt.Fprintln(os.Stderr, "This command requires root privileges.") + os.Exit(1) + } + cmd := installer.BootstrapCommand() + cmd.SilenceUsage = true + os.Exit(runcmd.Run(cmd)) +} diff --git a/cmd/installer/subcommands/daemon/api.go b/cmd/installer/subcommands/daemon/api.go index a55df5940aaf3..2b1eeb174a67f 100644 --- a/cmd/installer/subcommands/daemon/api.go +++ b/cmd/installer/subcommands/daemon/api.go @@ -91,7 +91,44 @@ func apiCommands(global *command.GlobalParams) []*cobra.Command { }) }, } - return []*cobra.Command{setCatalogCmd, startExperimentCmd, stopExperimentCmd, promoteExperimentCmd, installCmd} + startConfigExperimentCmd := &cobra.Command{ + Use: "start-config-experiment package version", + Aliases: []string{"start-config"}, + Short: "Starts an experiment", + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) error { + return experimentFxWrapper(startConfig, &cliParams{ + GlobalParams: *global, + pkg: args[0], + version: args[1], + }) + }, + } + stopConfigExperimentCmd := &cobra.Command{ + Use: "stop-config-experiment package", + Aliases: []string{"stop-config"}, + Short: "Stops an experiment", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return experimentFxWrapper(stopConfig, &cliParams{ + GlobalParams: *global, + pkg: args[0], + }) + }, + } + promoteConfigExperimentCmd := &cobra.Command{ + Use: "promote-config-experiment package", + Aliases: []string{"promote-config"}, + Short: "Promotes an experiment", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return experimentFxWrapper(promoteConfig, &cliParams{ + GlobalParams: *global, + pkg: args[0], + }) + }, + } + return []*cobra.Command{setCatalogCmd, startExperimentCmd, stopExperimentCmd, promoteExperimentCmd, installCmd, startConfigExperimentCmd, stopConfigExperimentCmd, promoteConfigExperimentCmd} } func experimentFxWrapper(f interface{}, params *cliParams) error { @@ -144,6 +181,33 @@ func promote(params *cliParams, client localapiclient.Component) error { return nil } +func startConfig(params *cliParams, client localapiclient.Component) error { + err := client.StartConfigExperiment(params.pkg, params.version) + if err != nil { + fmt.Println("Error starting config experiment:", err) + return err + } + return nil +} + +func stopConfig(params *cliParams, client localapiclient.Component) error { + err := client.StopConfigExperiment(params.pkg) + if err != nil { + fmt.Println("Error stopping config experiment:", err) + return err + } + return nil +} + +func promoteConfig(params *cliParams, client localapiclient.Component) error { + err := client.PromoteConfigExperiment(params.pkg) + if err != nil { + fmt.Println("Error promoting config experiment:", err) + return err + } + return nil +} + func install(params *cliParams, client localapiclient.Component) error { err := client.Install(params.pkg, params.version) if err != nil { diff --git a/cmd/installer/subcommands/daemon/run_windows.go b/cmd/installer/subcommands/daemon/run_windows.go index 4559addad1fe1..70205f2e2139b 100644 --- a/cmd/installer/subcommands/daemon/run_windows.go +++ b/cmd/installer/subcommands/daemon/run_windows.go @@ -27,7 +27,7 @@ type windowsService struct { func getFxOptions(global *command.GlobalParams) []fx.Option { return []fx.Option{ getCommonFxOption(global), - fxutil.FxAgentBase(true), + fxutil.FxAgentBase(), // Force the instantiation of some components fx.Invoke(func(_ pid.Component) {}), fx.Invoke(func(_ localapi.Component) {}), diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 32ba5484edd54..ae281fbeddc83 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -10,11 +10,12 @@ import ( "context" "fmt" "os" + "runtime" "strings" "time" "github.com/DataDog/datadog-agent/cmd/installer/command" - "github.com/DataDog/datadog-agent/pkg/fleet/bootstraper" + "github.com/DataDog/datadog-agent/pkg/fleet/bootstrapper" "github.com/DataDog/datadog-agent/pkg/fleet/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" @@ -22,6 +23,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/yaml.v2" ) const ( @@ -46,15 +48,24 @@ const ( envAgentDistChannel = "DD_AGENT_DIST_CHANNEL" ) +// BootstrapCommand returns the bootstrap command. +func BootstrapCommand() *cobra.Command { + return bootstrapCommand() +} + // Commands returns the installer subcommands. func Commands(_ *command.GlobalParams) []*cobra.Command { return []*cobra.Command{ bootstrapCommand(), installCommand(), + setupCommand(), removeCommand(), installExperimentCommand(), removeExperimentCommand(), promoteExperimentCommand(), + installConfigExperimentCommand(), + removeConfigExperimentCommand(), + promoteConfigExperimentCommand(), garbageCollectCommand(), purgeCommand(), isInstalledCommand(), @@ -119,11 +130,19 @@ func newInstallerCmd(operation string) (_ *installerCmd, err error) { }, nil } -type bootstraperCmd struct { +func (i *installerCmd) stop(err error) { + i.cmd.Stop(err) + err = i.Installer.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to close Installer: %v\n", err) + } +} + +type bootstrapperCmd struct { *cmd } -func newBootstraperCmd(operation string) *bootstraperCmd { +func newBootstrapperCmd(operation string) *bootstrapperCmd { cmd := newCmd(operation) cmd.span.SetTag("env.DD_UPGRADE", os.Getenv(envUpgrade)) cmd.span.SetTag("env.DD_APM_INSTRUMENTATION_NO_CONFIG_CHANGE", os.Getenv(envAPMInstrumentationNoConfigChange)) @@ -142,17 +161,45 @@ func newBootstraperCmd(operation string) *bootstraperCmd { cmd.span.SetTag("env.DD_RPM_REPO_GPGCHECK", os.Getenv(envRPMRepoGPGCheck)) cmd.span.SetTag("env.DD_AGENT_MAJOR_VERSION", os.Getenv(envAgentMajorVersion)) cmd.span.SetTag("env.DD_AGENT_MINOR_VERSION", os.Getenv(envAgentMinorVersion)) - return &bootstraperCmd{ + return &bootstrapperCmd{ cmd: cmd, } } +type telemetryConfigFields struct { + APIKey string `yaml:"api_key"` + Site string `yaml:"site"` +} + +// telemetryConfig is a best effort to get the API key / site from `datadog.yaml`. +func telemetryConfig() telemetryConfigFields { + configPath := "/etc/datadog-agent/datadog.yaml" + if runtime.GOOS == "windows" { + configPath = "C:\\ProgramData\\Datadog\\datadog.yaml" + } + rawConfig, err := os.ReadFile(configPath) + if err != nil { + return telemetryConfigFields{} + } + var config telemetryConfigFields + err = yaml.Unmarshal(rawConfig, &config) + if err != nil { + return telemetryConfigFields{} + } + return config +} + func newTelemetry(env *env.Env) *telemetry.Telemetry { - if env.APIKey == "" { - fmt.Printf("telemetry disabled: missing DD_API_KEY\n") - return nil + config := telemetryConfig() + apiKey := env.APIKey + if apiKey == "" { + apiKey = config.APIKey + } + site := env.Site + if site == "" { + site = config.Site } - t, err := telemetry.NewTelemetry(env, "datadog-installer") + t, err := telemetry.NewTelemetry(apiKey, site, "datadog-installer") if err != nil { fmt.Printf("failed to initialize telemetry: %v\n", err) return nil @@ -205,17 +252,35 @@ func bootstrapCommand() *cobra.Command { Short: "Bootstraps the package with the first version.", GroupID: "bootstrap", RunE: func(_ *cobra.Command, _ []string) (err error) { - b := newBootstraperCmd("bootstrap") + b := newBootstrapperCmd("bootstrap") defer func() { b.Stop(err) }() ctx, cancel := context.WithTimeout(b.ctx, timeout) defer cancel() - return bootstraper.Bootstrap(ctx, b.env) + return bootstrapper.Bootstrap(ctx, b.env) }, } cmd.Flags().DurationVarP(&timeout, "timeout", "T", 3*time.Minute, "timeout to bootstrap with") return cmd } +func setupCommand() *cobra.Command { + var timeout time.Duration + cmd := &cobra.Command{ + Use: "setup", + Hidden: true, + GroupID: "installer", + RunE: func(_ *cobra.Command, _ []string) (err error) { + cmd := newCmd("setup") + defer func() { cmd.Stop(err) }() + ctx, cancel := context.WithTimeout(cmd.ctx, timeout) + defer cancel() + return installer.Setup(ctx, cmd.env) + }, + } + cmd.Flags().DurationVarP(&timeout, "timeout", "T", 3*time.Minute, "timeout to install with") + return cmd +} + func installCommand() *cobra.Command { var installArgs []string cmd := &cobra.Command{ @@ -228,7 +293,7 @@ func installCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.span.SetTag("params.url", args[0]) return i.Install(i.ctx, args[0], installArgs) }, @@ -248,7 +313,7 @@ func removeCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.span.SetTag("params.package", args[0]) return i.Remove(i.ctx, args[0]) }, @@ -267,7 +332,7 @@ func purgeCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.Purge(i.ctx) return nil }, @@ -286,7 +351,7 @@ func installExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.span.SetTag("params.url", args[0]) return i.InstallExperiment(i.ctx, args[0]) }, @@ -305,7 +370,7 @@ func removeExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.span.SetTag("params.package", args[0]) return i.RemoveExperiment(i.ctx, args[0]) }, @@ -324,7 +389,7 @@ func promoteExperimentCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) i.span.SetTag("params.package", args[0]) return i.PromoteExperiment(i.ctx, args[0]) }, @@ -332,6 +397,64 @@ func promoteExperimentCommand() *cobra.Command { return cmd } +func installConfigExperimentCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "install-config-experiment ", + Short: "Install a config experiment", + GroupID: "installer", + Args: cobra.ExactArgs(2), + RunE: func(_ *cobra.Command, args []string) (err error) { + i, err := newInstallerCmd("install_config_experiment") + if err != nil { + return err + } + defer func() { i.Stop(err) }() + i.span.SetTag("params.package", args[0]) + i.span.SetTag("params.version", args[1]) + return i.InstallConfigExperiment(i.ctx, args[0], args[1]) + }, + } + return cmd +} + +func removeConfigExperimentCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "remove-config-experiment ", + Short: "Remove a config experiment", + GroupID: "installer", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) (err error) { + i, err := newInstallerCmd("remove_config_experiment") + if err != nil { + return err + } + defer func() { i.Stop(err) }() + i.span.SetTag("params.package", args[0]) + return i.RemoveConfigExperiment(i.ctx, args[0]) + }, + } + return cmd +} + +func promoteConfigExperimentCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "promote-config-experiment ", + Short: "Promote a config experiment", + GroupID: "installer", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) (err error) { + i, err := newInstallerCmd("promote_config_experiment") + if err != nil { + return err + } + defer func() { i.Stop(err) }() + i.span.SetTag("params.package", args[0]) + return i.PromoteConfigExperiment(i.ctx, args[0]) + }, + } + return cmd +} + func garbageCollectCommand() *cobra.Command { cmd := &cobra.Command{ Use: "garbage-collect", @@ -343,7 +466,7 @@ func garbageCollectCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) return i.GarbageCollect(i.ctx) }, } @@ -366,7 +489,7 @@ func isInstalledCommand() *cobra.Command { if err != nil { return err } - defer func() { i.Stop(err) }() + defer i.stop(err) installed, err := i.IsInstalled(i.ctx, args[0]) if err != nil { return err diff --git a/cmd/otel-agent/config/agent_config.go b/cmd/otel-agent/config/agent_config.go index 98695334c1b3f..edcca659ea329 100644 --- a/cmd/otel-agent/config/agent_config.go +++ b/cmd/otel-agent/config/agent_config.go @@ -8,6 +8,7 @@ package config import ( "context" + "errors" "fmt" "strings" @@ -59,8 +60,14 @@ var logLevelReverseMap = func(src map[string]logLevel) map[logLevel]string { return reverse }(logLevelMap) +// ErrNoDDExporter indicates there is no Datadog exporter in the configs +var ErrNoDDExporter = fmt.Errorf("no datadog exporter found") + // NewConfigComponent creates a new config component from the given URIs func NewConfigComponent(ctx context.Context, ddCfg string, uris []string) (config.Component, error) { + if len(uris) == 0 { + return nil, errors.New("no URIs provided for configs") + } // Load the configuration from the fileName rs := confmap.ResolverSettings{ URIs: uris, @@ -220,7 +227,7 @@ func getDDExporterConfig(cfg *confmap.Conf) (*datadogexporter.Config, error) { } } if len(configs) == 0 { - return nil, fmt.Errorf("no datadog exporter found") + return nil, ErrNoDDExporter } // Check if we have multiple datadog exporters // We only support one exporter for now diff --git a/cmd/otel-agent/config/agent_config_test.go b/cmd/otel-agent/config/agent_config_test.go index fc6b4dd6b21ff..c9f742b3406e8 100644 --- a/cmd/otel-agent/config/agent_config_test.go +++ b/cmd/otel-agent/config/agent_config_test.go @@ -10,10 +10,9 @@ import ( "fmt" "io/fs" "os" - "strings" "testing" - pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" + configmock "github.com/DataDog/datadog-agent/pkg/config/mock" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/stretchr/testify/assert" @@ -25,8 +24,12 @@ type ConfigTestSuite struct { } func (suite *ConfigTestSuite) SetupTest() { - datadog := pkgconfigmodel.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) - pkgconfigsetup.SetDatadog(datadog) + configmock.New(suite.T()) +} + +func TestNoURIsProvided(t *testing.T) { + _, err := NewConfigComponent(context.Background(), "", []string{}) + assert.Error(t, err, "no URIs provided for configs") } func (suite *ConfigTestSuite) TestAgentConfig() { diff --git a/cmd/otel-agent/main.go b/cmd/otel-agent/main.go index 716aafbe70023..9d75583cf85ce 100644 --- a/cmd/otel-agent/main.go +++ b/cmd/otel-agent/main.go @@ -14,6 +14,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/internal/runcmd" "github.com/DataDog/datadog-agent/cmd/otel-agent/command" "github.com/DataDog/datadog-agent/pkg/util/flavor" + _ "github.com/DataDog/datadog-agent/pkg/version" ) func main() { diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index 2373bc4453363..8fd1dd83ec30a 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "go.opentelemetry.io/collector/confmap" + "github.com/DataDog/datadog-agent/cmd/agent/common" agentConfig "github.com/DataDog/datadog-agent/cmd/otel-agent/config" "github.com/DataDog/datadog-agent/cmd/otel-agent/subcommands" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" @@ -37,7 +38,6 @@ import ( collectorcontribFx "github.com/DataDog/datadog-agent/comp/otelcol/collector-contrib/fx" collectordef "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" collectorfx "github.com/DataDog/datadog-agent/comp/otelcol/collector/fx" - configstorefx "github.com/DataDog/datadog-agent/comp/otelcol/configstore/fx" converter "github.com/DataDog/datadog-agent/comp/otelcol/converter/def" converterfx "github.com/DataDog/datadog-agent/comp/otelcol/converter/fx" "github.com/DataDog/datadog-agent/comp/otelcol/logsagentpipeline" @@ -92,7 +92,28 @@ func (o *orchestratorinterfaceimpl) Reset() { } func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, opts ...fx.Option) error { - err := fxutil.Run( + acfg, err := agentConfig.NewConfigComponent(context.Background(), params.CoreConfPath, params.ConfPaths) + if err != nil && err != agentConfig.ErrNoDDExporter { + return err + } + uris := append(params.ConfPaths, params.Sets...) + if err == agentConfig.ErrNoDDExporter { + return fxutil.Run( + fx.Supply(uris), + fx.Supply(optional.NewNoneOption[coreconfig.Component]()), + converterfx.Module(), + fx.Provide(func(cp converter.Component) confmap.Converter { + return cp + }), + collectorcontribFx.Module(), + collectorfx.ModuleNoAgent(), + fx.Options(opts...), + fx.Invoke(func(_ collectordef.Component) { + }), + ) + } + + return fxutil.Run( ForwarderBundle(), logtracefx.Module(), inventoryagentimpl.Module(), @@ -107,19 +128,16 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func(cp converter.Component) confmap.Converter { return cp }), - configstorefx.Module(), fx.Provide(func() (coreconfig.Component, error) { - c, err := agentConfig.NewConfigComponent(context.Background(), params.CoreConfPath, params.ConfPaths) - if err != nil { - return nil, err - } - pkgconfigenv.DetectFeatures(c) - return c, nil + pkgconfigenv.DetectFeatures(acfg) + return acfg, nil }), - workloadmetafx.Module(workloadmeta.NewParams()), - fx.Provide(func() []string { - return append(params.ConfPaths, params.Sets...) + fxutil.ProvideOptional[coreconfig.Component](), + workloadmetafx.Module(workloadmeta.Params{ + AgentType: workloadmeta.NodeAgent, + InitHelper: common.GetWorkloadmetaInit(), }), + fx.Supply(uris), fx.Provide(func(h hostnameinterface.Component) (serializerexporter.SourceProviderFunc, error) { return h.Get, nil }), @@ -165,7 +183,9 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) }), - fx.Provide(tagger.NewTaggerParams), + fx.Provide(func() tagger.Params { + return tagger.NewNodeRemoteTaggerParamsWithFallback() + }), taggerimpl.Module(), telemetryimpl.Module(), fx.Provide(func(cfg traceconfig.Component) telemetry.TelemetryCollector { @@ -193,10 +213,6 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, }), traceagentfx.Module(), ) - if err != nil { - return err - } - return nil } // ForwarderBundle returns the fx.Option for the forwarder bundle. diff --git a/cmd/otel-agent/subcommands/run/command_test.go b/cmd/otel-agent/subcommands/run/command_test.go index 89cbc6933a635..48ad0d876fbc0 100644 --- a/cmd/otel-agent/subcommands/run/command_test.go +++ b/cmd/otel-agent/subcommands/run/command_test.go @@ -15,10 +15,22 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) -func TestFxRun(t *testing.T) { +func TestFxRun_WithDatadogExporter(t *testing.T) { fxutil.TestRun(t, func() error { ctx := context.Background() - cliParams := &subcommands.GlobalParams{} - return runOTelAgentCommand(ctx, cliParams) + params := &subcommands.GlobalParams{ + ConfPaths: []string{"test_config.yaml"}, + } + return runOTelAgentCommand(ctx, params) + }) +} + +func TestFxRun_NoDatadogExporter(t *testing.T) { + fxutil.TestRun(t, func() error { + ctx := context.Background() + params := &subcommands.GlobalParams{ + ConfPaths: []string{"test_config_no_dd.yaml"}, + } + return runOTelAgentCommand(ctx, params) }) } diff --git a/cmd/otel-agent/subcommands/run/test_config.yaml b/cmd/otel-agent/subcommands/run/test_config.yaml new file mode 100644 index 0000000000000..d24e4565e93cf --- /dev/null +++ b/cmd/otel-agent/subcommands/run/test_config.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + http: + endpoint: "localhost:4318" + grpc: + endpoint: "localhost:4317" + +exporters: + datadog: + api: + key: "abc" + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [datadog] + telemetry: + metrics: + address: 127.0.0.1:8888 diff --git a/cmd/otel-agent/subcommands/run/test_config_no_dd.yaml b/cmd/otel-agent/subcommands/run/test_config_no_dd.yaml new file mode 100644 index 0000000000000..9b6dd21bcdb53 --- /dev/null +++ b/cmd/otel-agent/subcommands/run/test_config_no_dd.yaml @@ -0,0 +1,19 @@ +receivers: + otlp: + protocols: + http: + endpoint: "localhost:4318" + grpc: + endpoint: "localhost:4317" + +exporters: + debug: + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + telemetry: + metrics: + address: 127.0.0.1:8888 diff --git a/cmd/process-agent/command/main_common.go b/cmd/process-agent/command/main_common.go index cd0fc37d6cb33..7c68bf17a73de 100644 --- a/cmd/process-agent/command/main_common.go +++ b/cmd/process-agent/command/main_common.go @@ -59,6 +59,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/util" ddutil "github.com/DataDog/datadog-agent/pkg/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" + "github.com/DataDog/datadog-agent/pkg/util/fxutil/logging" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/version" @@ -184,7 +185,8 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error { }), // Provides specific features to our own fx wrapper (logging, lifecycle, shutdowner) - fxutil.FxAgentBase(true), + fxutil.FxAgentBase(), + logging.EnableFxLoggingOnDebug[logcomp.Component](), // Set the pid file path fx.Supply(pidimpl.NewParams(globalParams.PidFilePath)), diff --git a/cmd/security-agent/command/command.go b/cmd/security-agent/command/command.go index 216fd7682307b..23639e147fb20 100644 --- a/cmd/security-agent/command/command.go +++ b/cmd/security-agent/command/command.go @@ -13,7 +13,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - commonpath "github.com/DataDog/datadog-agent/cmd/agent/common/path" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" ) // GlobalParams contains the values of agent-global Cobra flags. @@ -37,11 +37,11 @@ const LoggerName = "SECURITY" var ( defaultSecurityAgentConfigFilePaths = []string{ - path.Join(commonpath.DefaultConfPath, "datadog.yaml"), - path.Join(commonpath.DefaultConfPath, "security-agent.yaml"), + path.Join(defaultpaths.ConfPath, "datadog.yaml"), + path.Join(defaultpaths.ConfPath, "security-agent.yaml"), } - defaultSysProbeConfPath = path.Join(commonpath.DefaultConfPath, "system-probe.yaml") + defaultSysProbeConfPath = path.Join(defaultpaths.ConfPath, "system-probe.yaml") ) // MakeCommand makes the top-level Cobra command for this command. diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index 2a1563f34f160..d2e6ab66b4441 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -15,7 +15,6 @@ import ( "go.uber.org/fx" - commonpath "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/cmd/internal/runcmd" "github.com/DataDog/datadog-agent/cmd/security-agent/command" saconfig "github.com/DataDog/datadog-agent/cmd/security-agent/config" @@ -51,6 +50,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/agent" "github.com/DataDog/datadog-agent/pkg/security/utils" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/startstop" @@ -63,10 +63,10 @@ type service struct { var ( defaultSecurityAgentConfigFilePaths = []string{ - path.Join(commonpath.DefaultConfPath, "datadog.yaml"), - path.Join(commonpath.DefaultConfPath, "security-agent.yaml"), + path.Join(defaultpaths.ConfPath, "datadog.yaml"), + path.Join(defaultpaths.ConfPath, "security-agent.yaml"), } - defaultSysProbeConfPath = path.Join(commonpath.DefaultConfPath, "system-probe.yaml") + defaultSysProbeConfPath = path.Join(defaultpaths.ConfPath, "system-probe.yaml") ) // Name returns the service name diff --git a/cmd/security-agent/subcommands/runtime/activity_dump.go b/cmd/security-agent/subcommands/runtime/activity_dump.go index 46465a0986dd2..514cdda1bc612 100644 --- a/cmd/security-agent/subcommands/runtime/activity_dump.go +++ b/cmd/security-agent/subcommands/runtime/activity_dump.go @@ -9,11 +9,13 @@ package runtime import ( + "encoding/json" "fmt" "os" "github.com/spf13/cobra" "go.uber.org/fx" + "gopkg.in/yaml.v2" "github.com/DataDog/datadog-agent/cmd/security-agent/command" "github.com/DataDog/datadog-agent/comp/core" @@ -23,6 +25,7 @@ import ( secagent "github.com/DataDog/datadog-agent/pkg/security/agent" secconfig "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/proto/api" + "github.com/DataDog/datadog-agent/pkg/security/secl/rules" activity_tree "github.com/DataDog/datadog-agent/pkg/security/security_profile/activity_tree" "github.com/DataDog/datadog-agent/pkg/security/security_profile/dump" "github.com/DataDog/datadog-agent/pkg/security/utils" @@ -57,6 +60,8 @@ func activityDumpCommands(globalParams *command.GlobalParams) []*cobra.Command { activityDumpCmd.AddCommand(listCommands(globalParams)...) activityDumpCmd.AddCommand(stopCommands(globalParams)...) activityDumpCmd.AddCommand(diffCommands(globalParams)...) + activityDumpCmd.AddCommand(activityDumpToWorkloadPolicyCommands(globalParams)...) + activityDumpCmd.AddCommand(activityDumpToSeccompProfileCommands(globalParams)...) return []*cobra.Command{activityDumpCmd} } @@ -615,3 +620,246 @@ func stopActivityDump(_ log.Component, _ config.Component, _ secrets.Component, fmt.Println("done!") return nil } + +type activityDumpToWorkloadPolicyCliParams struct { + *command.GlobalParams + + input string + output string + kill bool + allowlist bool + lineage bool + service string + imageName string + imageTag string + fim bool +} + +func activityDumpToWorkloadPolicyCommands(globalParams *command.GlobalParams) []*cobra.Command { + cliParams := &activityDumpToWorkloadPolicyCliParams{ + GlobalParams: globalParams, + } + + ActivityDumpWorkloadPolicyCmd := &cobra.Command{ + Use: "workload-policy", + Hidden: true, + Short: "convert an activity dump to a workload policy", + RunE: func(_ *cobra.Command, _ []string) error { + return fxutil.OneShot(activityDumpToWorkloadPolicy, + fx.Supply(cliParams), + fx.Supply(core.BundleParams{ + ConfigParams: config.NewSecurityAgentParams(globalParams.ConfigFilePaths), + SecretParams: secrets.NewEnabledParams(), + LogParams: log.ForOneShot(command.LoggerName, "info", true)}), + core.Bundle(), + ) + }, + } + + ActivityDumpWorkloadPolicyCmd.Flags().StringVar( + &cliParams.input, + "input", + "", + "path to the activity-dump file", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().StringVar( + &cliParams.output, + "output", + "", + "path to the generated workload policy file", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().BoolVar( + &cliParams.kill, + "kill", + false, + "generate kill action with the workload policy", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().BoolVar( + &cliParams.fim, + "fim", + false, + "generate fim rules with the workload policy", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().BoolVar( + &cliParams.allowlist, + "allowlist", + false, + "generate allow list rules", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().BoolVar( + &cliParams.lineage, + "lineage", + false, + "generate lineage rules", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().StringVar( + &cliParams.service, + "service", + "", + "apply on specified service", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().StringVar( + &cliParams.imageTag, + "image-tag", + "", + "apply on specified image tag", + ) + + ActivityDumpWorkloadPolicyCmd.Flags().StringVar( + &cliParams.imageName, + "image-name", + "", + "apply on specified image name", + ) + + return []*cobra.Command{ActivityDumpWorkloadPolicyCmd} +} + +func activityDumpToWorkloadPolicy(_ log.Component, _ config.Component, _ secrets.Component, args *activityDumpToWorkloadPolicyCliParams) error { + + opts := dump.SECLRuleOpts{ + EnableKill: args.kill, + AllowList: args.allowlist, + Lineage: args.lineage, + Service: args.service, + ImageName: args.imageName, + ImageTag: args.imageTag, + FIM: args.fim, + } + + ads, err := dump.LoadActivityDumpsFromFiles(args.input) + if err != nil { + return err + } + + generatedRules := dump.GenerateRules(ads, opts) + generatedRules = utils.BuildPatterns(generatedRules) + + policyDef := rules.PolicyDef{ + Rules: generatedRules, + } + + // Verify policy syntax + var policyName string + if len(args.imageName) > 0 { + policyName = fmt.Sprintf("%s_policy", args.imageName) + } else { + policyName = "workload_policy" + } + policy, err := rules.LoadPolicyFromDefinition(policyName, "workload", &policyDef, nil, nil) + + if err != nil { + return fmt.Errorf("error in generated ruleset's syntax: '%s'", err) + } + + b, err := yaml.Marshal(policy) + if err != nil { + return err + } + + output := os.Stdout + if args.output != "" && args.output != "-" { + output, err = os.Create(args.output) + if err != nil { + return err + } + defer output.Close() + } + + fmt.Fprint(output, string(b)) + + return nil +} + +type activityDumpToSeccompProfileCliParams struct { + *command.GlobalParams + + input string + output string + format string +} + +func activityDumpToSeccompProfileCommands(globalParams *command.GlobalParams) []*cobra.Command { + cliParams := &activityDumpToSeccompProfileCliParams{ + GlobalParams: globalParams, + } + + ActivityDumpToSeccompProfileCmd := &cobra.Command{ + Use: "workload-seccomp", + Hidden: true, + Short: "convert an activity dump to a seccomp profile", + RunE: func(_ *cobra.Command, _ []string) error { + return fxutil.OneShot(activityDumpToSeccompProfile, + fx.Supply(cliParams), + fx.Supply(core.BundleParams{ + ConfigParams: config.NewSecurityAgentParams(globalParams.ConfigFilePaths), + SecretParams: secrets.NewEnabledParams(), + LogParams: log.ForOneShot(command.LoggerName, "info", true)}), + core.Bundle(), + ) + }, + } + + ActivityDumpToSeccompProfileCmd.Flags().StringVar( + &cliParams.input, + "input", + "", + "path to the activity-dump file", + ) + + ActivityDumpToSeccompProfileCmd.Flags().StringVar( + &cliParams.output, + "output", + "", + "path to the generated seccomp profile file", + ) + + ActivityDumpToSeccompProfileCmd.Flags().StringVar( + &cliParams.format, + "format", + "json", + "format of the generated seccomp profile file", + ) + + return []*cobra.Command{ActivityDumpToSeccompProfileCmd} +} +func activityDumpToSeccompProfile(_ log.Component, _ config.Component, _ secrets.Component, args *activityDumpToSeccompProfileCliParams) error { + + ads, err := dump.LoadActivityDumpsFromFiles(args.input) + if err != nil { + return err + } + + seccompProfile := dump.GenerateSeccompProfile(ads) + + var b []byte + if args.format == "yaml" { + b, err = yaml.Marshal(seccompProfile) + } else { + b, err = json.Marshal(seccompProfile) + } + + if err != nil { + return err + } + + output := os.Stdout + if args.output != "" && args.output != "-" { + output, err = os.Create(args.output) + if err != nil { + return err + } + defer output.Close() + } + + fmt.Fprint(output, string(b)) + + return nil +} diff --git a/cmd/security-agent/subcommands/runtime/activity_dump_test.go b/cmd/security-agent/subcommands/runtime/activity_dump_test.go index ed1bca1cdf6f1..555461135dee5 100644 --- a/cmd/security-agent/subcommands/runtime/activity_dump_test.go +++ b/cmd/security-agent/subcommands/runtime/activity_dump_test.go @@ -53,3 +53,19 @@ func TestDumpActivityDumpCommand(t *testing.T) { generateActivityDump, func() {}) } + +func TestActivityDumpToWorkloadPolicyCommand(t *testing.T) { + fxutil.TestOneShotSubcommand(t, + Commands(&command.GlobalParams{}), + []string{"runtime", "activity-dump", "workload-policy", "--input", "file"}, + activityDumpToWorkloadPolicy, + func() {}) +} + +func TestActivityDumpToSeccompProfileCommand(t *testing.T) { + fxutil.TestOneShotSubcommand(t, + Commands(&command.GlobalParams{}), + []string{"runtime", "activity-dump", "workload-seccomp", "--input", "file", "--output", "file"}, + activityDumpToSeccompProfile, + func() {}) +} diff --git a/cmd/security-agent/subcommands/runtime/security_profile.go b/cmd/security-agent/subcommands/runtime/security_profile.go index 27d65dad73854..e02a85c37cca7 100644 --- a/cmd/security-agent/subcommands/runtime/security_profile.go +++ b/cmd/security-agent/subcommands/runtime/security_profile.go @@ -41,14 +41,14 @@ func securityProfileCommands(globalParams *command.GlobalParams) []*cobra.Comman Short: "security profile commands", } - securityProfileCmd.AddCommand(securityProfileShowCommands(globalParams)...) + securityProfileCmd.AddCommand(showSecurityProfileCommands(globalParams)...) securityProfileCmd.AddCommand(listSecurityProfileCommands(globalParams)...) securityProfileCmd.AddCommand(saveSecurityProfileCommands(globalParams)...) return []*cobra.Command{securityProfileCmd} } -func securityProfileShowCommands(globalParams *command.GlobalParams) []*cobra.Command { +func showSecurityProfileCommands(globalParams *command.GlobalParams) []*cobra.Command { cliParams := &securityProfileCliParams{ GlobalParams: globalParams, } diff --git a/cmd/serverless-init/cloudservice/cloudrun.go b/cmd/serverless-init/cloudservice/cloudrun.go index d664fb2f7cc93..120f3725367b8 100644 --- a/cmd/serverless-init/cloudservice/cloudrun.go +++ b/cmd/serverless-init/cloudservice/cloudrun.go @@ -6,21 +6,28 @@ package cloudservice import ( + "fmt" + "github.com/DataDog/datadog-agent/pkg/util/log" "os" "github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice/helper" ) const ( - revisionNameEnvVar = "K_REVISION" //nolint:revive // TODO(SERV) Fix revive linter - ServiceNameEnvVar = "K_SERVICE" + revisionNameEnvVar = "K_REVISION" + ServiceNameEnvVar = "K_SERVICE" // ServiceNameEnvVar is also used in the trace package + configurationNameEnvVar = "K_CONFIGURATION" + functionTypeEnvVar = "FUNCTION_SIGNATURE_TYPE" + functionTargetEnvVar = "FUNCTION_TARGET" // exists as a cloudrunfunction env var for all runtimes except Go ) var metadataHelperFunc = helper.GetMetaData // CloudRun has helper functions for getting Google Cloud Run data -type CloudRun struct{} +type CloudRun struct { + cloudRunFunctionMode bool +} // GetTags returns a map of gcp-related tags. func (c *CloudRun) GetTags() map[string]string { @@ -28,6 +35,7 @@ func (c *CloudRun) GetTags() map[string]string { revisionName := os.Getenv(revisionNameEnvVar) serviceName := os.Getenv(ServiceNameEnvVar) + configName := os.Getenv(configurationNameEnvVar) if revisionName != "" { tags["revision_name"] = revisionName @@ -37,21 +45,48 @@ func (c *CloudRun) GetTags() map[string]string { tags["service_name"] = serviceName } + if configName != "" { + tags["configuration_name"] = configName + } + + if c.cloudRunFunctionMode { + tags = getFunctionTags(tags) + } tags["origin"] = c.GetOrigin() tags["_dd.origin"] = c.GetOrigin() return tags } +func getFunctionTags(tags map[string]string) map[string]string { + functionTarget := os.Getenv(functionTargetEnvVar) + functionSignatureType := os.Getenv(functionTypeEnvVar) + + if functionTarget != "" { + tags["function_target"] = functionTarget + } + + if functionSignatureType != "" { + tags["function_signature_type"] = functionSignatureType + } + return tags +} + // GetOrigin returns the `origin` attribute type for the given // cloud service. func (c *CloudRun) GetOrigin() string { + if c.cloudRunFunctionMode { + return "cloudfunctions" + } return "cloudrun" } // GetPrefix returns the prefix that we're prefixing all // metrics with. func (c *CloudRun) GetPrefix() string { + if c.cloudRunFunctionMode { + return "gcp.cloudfunctions" + } return "gcp.run" } @@ -64,3 +99,9 @@ func isCloudRunService() bool { _, exists := os.LookupEnv(ServiceNameEnvVar) return exists } + +func isCloudRunFunction() bool { + _, cloudRunFunctionMode := os.LookupEnv(functionTargetEnvVar) + log.Debug(fmt.Sprintf("cloud function mode SET TO: %t", cloudRunFunctionMode)) + return cloudRunFunctionMode +} diff --git a/cmd/serverless-init/cloudservice/cloudrun_test.go b/cmd/serverless-init/cloudservice/cloudrun_test.go index 4832b30f48478..41babaa632a1c 100644 --- a/cmd/serverless-init/cloudservice/cloudrun_test.go +++ b/cmd/serverless-init/cloudservice/cloudrun_test.go @@ -78,3 +78,45 @@ func TestGetCloudRunTagsWithEnvironmentVariables(t *testing.T) { "_dd.origin": "cloudrun", }, tags) } + +func TestGetCloudRunFunctionTagsWithEnvironmentVariables(t *testing.T) { + service := &CloudRun{cloudRunFunctionMode: true} + + metadataHelperFunc = func(*helper.GCPConfig) *helper.GCPMetadata { + return &helper.GCPMetadata{ + ContainerID: &helper.Info{ + TagName: "container_id", + Value: "test_container", + }, + Region: &helper.Info{ + TagName: "region", + Value: "test_region", + }, + ProjectID: &helper.Info{ + TagName: "project_id", + Value: "test_project", + }, + } + } + + t.Setenv("K_SERVICE", "test_service") + t.Setenv("K_REVISION", "test_revision") + t.Setenv("K_CONFIGURATION", "test_config") + t.Setenv("FUNCTION_SIGNATURE_TYPE", "test_signature") + t.Setenv("FUNCTION_TARGET", "test_target") + + tags := service.GetTags() + + assert.Equal(t, map[string]string{ + "container_id": "test_container", + "region": "test_region", + "origin": "cloudfunctions", + "project_id": "test_project", + "service_name": "test_service", + "revision_name": "test_revision", + "configuration_name": "test_config", + "_dd.origin": "cloudfunctions", + "function_target": "test_target", + "function_signature_type": "test_signature", + }, tags) +} diff --git a/cmd/serverless-init/cloudservice/service.go b/cmd/serverless-init/cloudservice/service.go index 0449d28d51533..54c9a3576a35a 100644 --- a/cmd/serverless-init/cloudservice/service.go +++ b/cmd/serverless-init/cloudservice/service.go @@ -51,6 +51,9 @@ func (l *LocalService) Init() error { //nolint:revive // TODO(SERV) Fix revive linter func GetCloudServiceType() CloudService { if isCloudRunService() { + if isCloudRunFunction() { + return &CloudRun{cloudRunFunctionMode: true} + } return &CloudRun{} } diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index f11c59911eadd..90772c5f91fa6 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -62,7 +62,6 @@ github.com/DataDog/appsec-internal-go/httpsec github.com/DataDog/appsec-internal-go/limiter github.com/DataDog/appsec-internal-go/log github.com/DataDog/appsec-internal-go/netip -github.com/DataDog/datadog-agent/cmd/agent/common/path github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice/helper github.com/DataDog/datadog-agent/comp/api/api/def @@ -141,9 +140,12 @@ github.com/DataDog/datadog-agent/pkg/collector/check/id github.com/DataDog/datadog-agent/pkg/collector/check/stats github.com/DataDog/datadog-agent/pkg/config/env github.com/DataDog/datadog-agent/pkg/config/model +github.com/DataDog/datadog-agent/pkg/config/nodetreemodel github.com/DataDog/datadog-agent/pkg/config/remote/data github.com/DataDog/datadog-agent/pkg/config/setup github.com/DataDog/datadog-agent/pkg/config/setup/constants +github.com/DataDog/datadog-agent/pkg/config/structure +github.com/DataDog/datadog-agent/pkg/config/teeconfig github.com/DataDog/datadog-agent/pkg/config/utils github.com/DataDog/datadog-agent/pkg/diagnose/diagnosis github.com/DataDog/datadog-agent/pkg/errors @@ -246,6 +248,7 @@ github.com/DataDog/datadog-agent/pkg/trace/stats github.com/DataDog/datadog-agent/pkg/trace/telemetry github.com/DataDog/datadog-agent/pkg/trace/timing github.com/DataDog/datadog-agent/pkg/trace/traceutil +github.com/DataDog/datadog-agent/pkg/trace/transform github.com/DataDog/datadog-agent/pkg/trace/version github.com/DataDog/datadog-agent/pkg/trace/watchdog github.com/DataDog/datadog-agent/pkg/trace/writer @@ -262,6 +265,7 @@ github.com/DataDog/datadog-agent/pkg/util/common github.com/DataDog/datadog-agent/pkg/util/containers github.com/DataDog/datadog-agent/pkg/util/containers/image github.com/DataDog/datadog-agent/pkg/util/containers/metrics/provider +github.com/DataDog/datadog-agent/pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/dmi github.com/DataDog/datadog-agent/pkg/util/docker github.com/DataDog/datadog-agent/pkg/util/ec2 @@ -407,11 +411,13 @@ github.com/aws/smithy-go/endpoints github.com/aws/smithy-go/internal/sync/singleflight github.com/aws/smithy-go/io github.com/aws/smithy-go/logging +github.com/aws/smithy-go/metrics github.com/aws/smithy-go/middleware github.com/aws/smithy-go/private/requestcompression github.com/aws/smithy-go/ptr github.com/aws/smithy-go/rand github.com/aws/smithy-go/time +github.com/aws/smithy-go/tracing github.com/aws/smithy-go/transport/http github.com/aws/smithy-go/transport/http/internal/io github.com/benbjohnson/clock @@ -663,6 +669,7 @@ go.opentelemetry.io/contrib/config go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconvutil go.opentelemetry.io/contrib/propagators/b3 @@ -725,7 +732,6 @@ go.opentelemetry.io/otel/sdk/trace go.opentelemetry.io/otel/sdk/trace/tracetest go.opentelemetry.io/otel/semconv/v1.17.0 go.opentelemetry.io/otel/semconv/v1.20.0 -go.opentelemetry.io/otel/semconv/v1.24.0 go.opentelemetry.io/otel/semconv/v1.25.0 go.opentelemetry.io/otel/semconv/v1.26.0 go.opentelemetry.io/otel/trace diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index 15d7cfd36fdb5..6734f6d29fec8 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -62,7 +62,6 @@ github.com/DataDog/appsec-internal-go/httpsec github.com/DataDog/appsec-internal-go/limiter github.com/DataDog/appsec-internal-go/log github.com/DataDog/appsec-internal-go/netip -github.com/DataDog/datadog-agent/cmd/agent/common/path github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice/helper github.com/DataDog/datadog-agent/comp/api/api/def @@ -141,9 +140,12 @@ github.com/DataDog/datadog-agent/pkg/collector/check/id github.com/DataDog/datadog-agent/pkg/collector/check/stats github.com/DataDog/datadog-agent/pkg/config/env github.com/DataDog/datadog-agent/pkg/config/model +github.com/DataDog/datadog-agent/pkg/config/nodetreemodel github.com/DataDog/datadog-agent/pkg/config/remote/data github.com/DataDog/datadog-agent/pkg/config/setup github.com/DataDog/datadog-agent/pkg/config/setup/constants +github.com/DataDog/datadog-agent/pkg/config/structure +github.com/DataDog/datadog-agent/pkg/config/teeconfig github.com/DataDog/datadog-agent/pkg/config/utils github.com/DataDog/datadog-agent/pkg/diagnose/diagnosis github.com/DataDog/datadog-agent/pkg/errors @@ -246,6 +248,7 @@ github.com/DataDog/datadog-agent/pkg/trace/stats github.com/DataDog/datadog-agent/pkg/trace/telemetry github.com/DataDog/datadog-agent/pkg/trace/timing github.com/DataDog/datadog-agent/pkg/trace/traceutil +github.com/DataDog/datadog-agent/pkg/trace/transform github.com/DataDog/datadog-agent/pkg/trace/version github.com/DataDog/datadog-agent/pkg/trace/watchdog github.com/DataDog/datadog-agent/pkg/trace/writer @@ -262,6 +265,7 @@ github.com/DataDog/datadog-agent/pkg/util/common github.com/DataDog/datadog-agent/pkg/util/containers github.com/DataDog/datadog-agent/pkg/util/containers/image github.com/DataDog/datadog-agent/pkg/util/containers/metrics/provider +github.com/DataDog/datadog-agent/pkg/util/defaultpaths github.com/DataDog/datadog-agent/pkg/util/dmi github.com/DataDog/datadog-agent/pkg/util/docker github.com/DataDog/datadog-agent/pkg/util/ec2 @@ -407,11 +411,13 @@ github.com/aws/smithy-go/endpoints github.com/aws/smithy-go/internal/sync/singleflight github.com/aws/smithy-go/io github.com/aws/smithy-go/logging +github.com/aws/smithy-go/metrics github.com/aws/smithy-go/middleware github.com/aws/smithy-go/private/requestcompression github.com/aws/smithy-go/ptr github.com/aws/smithy-go/rand github.com/aws/smithy-go/time +github.com/aws/smithy-go/tracing github.com/aws/smithy-go/transport/http github.com/aws/smithy-go/transport/http/internal/io github.com/benbjohnson/clock @@ -662,6 +668,7 @@ go.opentelemetry.io/contrib/config go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/request go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconv go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconvutil go.opentelemetry.io/contrib/propagators/b3 @@ -724,7 +731,6 @@ go.opentelemetry.io/otel/sdk/trace go.opentelemetry.io/otel/sdk/trace/tracetest go.opentelemetry.io/otel/semconv/v1.17.0 go.opentelemetry.io/otel/semconv/v1.20.0 -go.opentelemetry.io/otel/semconv/v1.24.0 go.opentelemetry.io/otel/semconv/v1.25.0 go.opentelemetry.io/otel/semconv/v1.26.0 go.opentelemetry.io/otel/trace diff --git a/cmd/system-probe/api/client/client.go b/cmd/system-probe/api/client/client.go index 2d7ed13f17af4..81f8efd38bebb 100644 --- a/cmd/system-probe/api/client/client.go +++ b/cmd/system-probe/api/client/client.go @@ -7,25 +7,10 @@ package client import ( - "context" - "net" "net/http" - "time" ) // Get returns a http client configured to talk to the system-probe func Get(socketPath string) *http.Client { - return &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 2, - IdleConnTimeout: 30 * time.Second, - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial(netType, socketPath) - }, - TLSHandshakeTimeout: 1 * time.Second, - ResponseHeaderTimeout: 5 * time.Second, - ExpectContinueTimeout: 50 * time.Millisecond, - }, - } + return newSystemProbeClient(socketPath) } diff --git a/cmd/system-probe/api/client/client_linux.go b/cmd/system-probe/api/client/client_linux.go index 92a558d19e4a2..b644838e80544 100644 --- a/cmd/system-probe/api/client/client_linux.go +++ b/cmd/system-probe/api/client/client_linux.go @@ -7,6 +7,31 @@ package client +import ( + "context" + "net" + "net/http" + "time" +) + const ( netType = "unix" ) + +// newSystemProbeClient returns a http client configured to talk to the system-probe +// Linux is unable to import pkg/process/net due to size restrictions. +func newSystemProbeClient(socketPath string) *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 2, + IdleConnTimeout: 30 * time.Second, + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial(netType, socketPath) + }, + TLSHandshakeTimeout: 1 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + ExpectContinueTimeout: 50 * time.Millisecond, + }, + } +} diff --git a/cmd/system-probe/api/client/client_others.go b/cmd/system-probe/api/client/client_others.go index 48ac5dcd6614a..1204bb37e7d1a 100644 --- a/cmd/system-probe/api/client/client_others.go +++ b/cmd/system-probe/api/client/client_others.go @@ -7,6 +7,30 @@ package client +import ( + "context" + "net" + "net/http" + "time" +) + const ( netType = "tcp" ) + +// newSystemProbeClient returns a http client configured to talk to the system-probe +func newSystemProbeClient(socketPath string) *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 2, + IdleConnTimeout: 30 * time.Second, + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial(netType, socketPath) + }, + TLSHandshakeTimeout: 1 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + ExpectContinueTimeout: 50 * time.Millisecond, + }, + } +} diff --git a/cmd/system-probe/api/client/client_windows.go b/cmd/system-probe/api/client/client_windows.go index d8217bb143e76..d4649c90bd5c6 100644 --- a/cmd/system-probe/api/client/client_windows.go +++ b/cmd/system-probe/api/client/client_windows.go @@ -7,6 +7,15 @@ package client -const ( - netType = "tcp" +import ( + "net/http" + + processNet "github.com/DataDog/datadog-agent/pkg/process/net" ) + +// newSystemProbeClient returns a http client configured to talk to the system-probe +// This is a simple wrapper around process_net.NewSystemProbeHttpClient because +// Linux is unable to import pkg/process/net due to size restrictions. +func newSystemProbeClient(_ string) *http.Client { + return processNet.NewSystemProbeClient() +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index 8ab6909d3db54..a2d20995b9804 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -27,9 +27,9 @@ import ( // StartServer starts the HTTP and gRPC servers for the system-probe, which registers endpoints from all enabled modules. func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmeta workloadmeta.Component, settings settings.Component) error { - conn, err := net.NewListener(cfg.SocketAddress) + conn, err := net.NewSystemProbeListener(cfg.SocketAddress) if err != nil { - return fmt.Errorf("error creating IPC socket: %s", err) + return err } mux := gorilla.NewRouter() diff --git a/cmd/system-probe/config/config.go b/cmd/system-probe/config/config.go index b592cc2f65f7f..1ae05b24a411b 100644 --- a/cmd/system-probe/config/config.go +++ b/cmd/system-probe/config/config.go @@ -43,6 +43,7 @@ const ( PingModule types.ModuleName = "ping" TracerouteModule types.ModuleName = "traceroute" DiscoveryModule types.ModuleName = "discovery" + GPUMonitoringModule types.ModuleName = "gpu" ) // New creates a config object for system-probe. It assumes no configuration has been loaded as this point. @@ -121,8 +122,9 @@ func load() (*types.Config, error) { npmEnabled := cfg.GetBool(netNS("enabled")) usmEnabled := cfg.GetBool(smNS("enabled")) ccmEnabled := cfg.GetBool(ccmNS("enabled")) + csmEnabled := cfg.GetBool(secNS("enabled")) - if npmEnabled || usmEnabled || ccmEnabled { + if npmEnabled || usmEnabled || ccmEnabled || (csmEnabled && cfg.GetBool(secNS("network_monitoring.enabled"))) { c.EnabledModules[NetworkTracerModule] = struct{}{} } if cfg.GetBool(spNS("enable_tcp_queue_length")) { @@ -161,6 +163,10 @@ func load() (*types.Config, error) { if cfg.GetBool(discoveryNS("enabled")) { c.EnabledModules[DiscoveryModule] = struct{}{} } + if cfg.GetBool(gpuNS("enabled")) { + c.EnabledModules[GPUMonitoringModule] = struct{}{} + } + if cfg.GetBool(wcdNS("enabled")) { c.EnabledModules[WindowsCrashDetectModule] = struct{}{} } diff --git a/cmd/system-probe/config/config_linux_test.go b/cmd/system-probe/config/config_linux_test.go index aab12a5c52ed5..883edaaab88c9 100644 --- a/cmd/system-probe/config/config_linux_test.go +++ b/cmd/system-probe/config/config_linux_test.go @@ -73,17 +73,26 @@ func TestEventStreamEnabledForSupportedKernelsLinux(t *testing.T) { func TestNPMEnabled(t *testing.T) { tests := []struct { - npm, usm, ccm bool - npmEnabled bool + npm, usm, ccm, csm, csmNpm bool + npmEnabled bool }{ - {false, false, false, false}, - {false, false, true, true}, - {false, true, false, true}, - {false, true, true, true}, - {true, false, false, true}, - {true, false, true, true}, - {true, true, false, true}, - {true, true, true, true}, + {false, false, false, false, false, false}, + {false, false, true, false, false, true}, + {false, true, false, false, false, true}, + {false, true, true, false, false, true}, + {true, false, false, false, false, true}, + {true, false, true, false, false, true}, + {true, true, false, false, false, true}, + {true, true, true, false, false, true}, + {false, false, false, true, false, false}, + {false, false, false, true, true, true}, + {false, false, true, true, false, true}, + {false, true, false, true, false, true}, + {false, true, true, true, false, true}, + {true, false, false, true, false, true}, + {true, false, true, true, false, true}, + {true, true, false, true, false, true}, + {true, true, true, true, false, true}, } mock.NewSystemProbe(t) @@ -92,6 +101,8 @@ func TestNPMEnabled(t *testing.T) { t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLED", strconv.FormatBool(te.npm)) t.Setenv("DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED", strconv.FormatBool(te.usm)) t.Setenv("DD_CCM_NETWORK_CONFIG_ENABLED", strconv.FormatBool(te.ccm)) + t.Setenv("DD_RUNTIME_SECURITY_CONFIG_ENABLED", strconv.FormatBool(te.csm)) + t.Setenv("DD_RUNTIME_SECURITY_CONFIG_NETWORK_MONITORING_ENABLED", strconv.FormatBool(te.csmNpm)) cfg, err := New("", "") require.NoError(t, err) assert.Equal(t, te.npmEnabled, cfg.ModuleIsEnabled(NetworkTracerModule), "unexpected network tracer module enablement: npm: %v, usm: %v, ccm: %v", te.npm, te.usm, te.ccm) diff --git a/cmd/system-probe/config/ns.go b/cmd/system-probe/config/ns.go index c6d098febcb5f..913b86b7915f5 100644 --- a/cmd/system-probe/config/ns.go +++ b/cmd/system-probe/config/ns.go @@ -65,3 +65,8 @@ func tracerouteNS(k ...string) string { func discoveryNS(k ...string) string { return nskey("discovery", k...) } + +// gpuNS adds `gpu_monitoring` namespace to config key +func gpuNS(k ...string) string { + return nskey("gpu_monitoring", k...) +} diff --git a/cmd/system-probe/modules/all_linux.go b/cmd/system-probe/modules/all_linux.go index 23f5a33e51119..869cda408a8d4 100644 --- a/cmd/system-probe/modules/all_linux.go +++ b/cmd/system-probe/modules/all_linux.go @@ -24,11 +24,13 @@ var All = []module.Factory{ // so EventMonitor has to follow NetworkTracer EventMonitor, Process, + DynamicInstrumentation, LanguageDetectionModule, ComplianceModule, Pinger, Traceroute, DiscoveryModule, + GPUMonitoring, } func inactivityEventLog(_ time.Duration) { diff --git a/cmd/system-probe/modules/all_linux_arm64.go b/cmd/system-probe/modules/all_linux_arm64.go index 689933bac8517..9115d7037b6c3 100644 --- a/cmd/system-probe/modules/all_linux_arm64.go +++ b/cmd/system-probe/modules/all_linux_arm64.go @@ -30,6 +30,7 @@ var All = []module.Factory{ Pinger, Traceroute, DiscoveryModule, + GPUMonitoring, } func inactivityEventLog(_ time.Duration) { diff --git a/cmd/system-probe/modules/gpu_monitoring.go b/cmd/system-probe/modules/gpu_monitoring.go new file mode 100644 index 0000000000000..5db344197a37a --- /dev/null +++ b/cmd/system-probe/modules/gpu_monitoring.go @@ -0,0 +1,81 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package modules + +import ( + "fmt" + "net/http" + "time" + + "go.uber.org/atomic" + + "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" + "github.com/DataDog/datadog-agent/cmd/system-probe/config" + sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" + "github.com/DataDog/datadog-agent/cmd/system-probe/utils" + "github.com/DataDog/datadog-agent/pkg/gpu" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +var _ module.Module = &GPUMonitoringModule{} +var gpuMonitoringConfigNamespaces = []string{gpu.GPUConfigNS} + +// GPUMonitoring Factory +var GPUMonitoring = module.Factory{ + Name: config.GPUMonitoringModule, + ConfigNamespaces: gpuMonitoringConfigNamespaces, + Fn: func(_ *sysconfigtypes.Config, _ module.FactoryDependencies) (module.Module, error) { + t, err := gpu.NewProbe(gpu.NewConfig(), nil) + if err != nil { + return nil, fmt.Errorf("unable to start GPU monitoring: %w", err) + } + + return &GPUMonitoringModule{ + Probe: t, + lastCheck: atomic.NewInt64(0), + }, nil + }, + NeedsEBPF: func() bool { + return true + }, +} + +// GPUMonitoringModule is a module for GPU monitoring +type GPUMonitoringModule struct { + *gpu.Probe + lastCheck *atomic.Int64 +} + +// Register registers the GPU monitoring module +func (t *GPUMonitoringModule) Register(httpMux *module.Router) error { + httpMux.HandleFunc("/check", func(w http.ResponseWriter, _ *http.Request) { + t.lastCheck.Store(time.Now().Unix()) + stats, err := t.Probe.GetAndFlush() + if err != nil { + log.Errorf("Error getting GPU stats: %v", err) + w.WriteHeader(500) + return + } + + utils.WriteAsJSON(w, stats) + }) + + return nil +} + +// GetStats returns the last check time +func (t *GPUMonitoringModule) GetStats() map[string]interface{} { + return map[string]interface{}{ + "last_check": t.lastCheck.Load(), + } +} + +// Close closes the GPU monitoring module +func (t *GPUMonitoringModule) Close() { + t.Probe.Close() +} diff --git a/cmd/system-probe/subcommands/config/command.go b/cmd/system-probe/subcommands/config/command.go index 0d9d81dcb47ce..7010d214143d5 100644 --- a/cmd/system-probe/subcommands/config/command.go +++ b/cmd/system-probe/subcommands/config/command.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl" "github.com/DataDog/datadog-agent/pkg/api/util" - "github.com/DataDog/datadog-agent/pkg/config/fetcher" + fetcher "github.com/DataDog/datadog-agent/pkg/config/fetcher/sysprobe" "github.com/DataDog/datadog-agent/pkg/config/settings" settingshttp "github.com/DataDog/datadog-agent/pkg/config/settings/http" "github.com/DataDog/datadog-agent/pkg/util/fxutil" diff --git a/cmd/systray/command/command.go b/cmd/systray/command/command.go index cd421597483e1..c237b0aa65bb3 100644 --- a/cmd/systray/command/command.go +++ b/cmd/systray/command/command.go @@ -14,11 +14,9 @@ import ( "strings" "github.com/spf13/cobra" - "go.uber.org/fx" "golang.org/x/sys/windows" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl" authtokenimpl "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/collector/collector" @@ -33,6 +31,7 @@ import ( "github.com/DataDog/datadog-agent/comp/systray/systray" "github.com/DataDog/datadog-agent/comp/systray/systray/systrayimpl" "github.com/DataDog/datadog-agent/pkg/serializer" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/winutil" @@ -93,18 +92,18 @@ func MakeCommand() *cobra.Command { return fxutil.Run( // core fx.Supply(core.BundleParams{ - ConfigParams: config.NewParams(path.DefaultConfPath), + ConfigParams: config.NewParams(defaultpaths.ConfPath), LogParams: logParams, }), core.Bundle(), // flare flare.Module(flare.NewParams( - path.GetDistPath(), - path.PyChecksPath, - path.DefaultLogFile, - path.DefaultJmxLogFile, - path.DefaultDogstatsDLogFile, - path.DefaultStreamlogsLogFile, + defaultpaths.GetDistPath(), + defaultpaths.PyChecksPath, + defaultpaths.LogFile, + defaultpaths.JmxLogFile, + defaultpaths.DogstatsDLogFile, + defaultpaths.StreamlogsLogFile, )), fx.Supply(optional.NewNoneOption[autodiscovery.Component]()), fx.Supply(optional.NewNoneOption[workloadmeta.Component]()), diff --git a/cmd/trace-agent/test/testsuite/traces_test.go b/cmd/trace-agent/test/testsuite/traces_test.go index 1404d6b558b12..93790de58b6ef 100644 --- a/cmd/trace-agent/test/testsuite/traces_test.go +++ b/cmd/trace-agent/test/testsuite/traces_test.go @@ -184,6 +184,34 @@ func TestTraces(t *testing.T) { }) }) + t.Run("normalize, obfuscate, sqllexer", func(t *testing.T) { + if err := r.RunAgent([]byte("apm_config:\r\n features:[\"sqllexer\"]\r\n")); err != nil { + t.Fatal(err) + } + defer r.KillAgent() + + p := testutil.GeneratePayload(1, &testutil.TraceConfig{ + MinSpans: 4, + Keep: true, + }, nil) + for _, span := range p[0] { + span.Service = strings.Repeat("a", 200) // Too long + span.Name = strings.Repeat("b", 200) // Too long + } + p[0][0].Type = "sql" + p[0][0].Resource = "SELECT secret FROM users WHERE id = 123" + if err := r.Post(p); err != nil { + t.Fatal(err) + } + waitForTrace(t, &r, func(v *pb.AgentPayload) { + assert.Equal(t, "SELECT secret FROM users WHERE id = ?", v.TracerPayloads[0].Chunks[0].Spans[0].Resource) + for _, s := range v.TracerPayloads[0].Chunks[0].Spans { + assert.Len(t, s.Service, 100) + assert.Len(t, s.Name, 100) + } + }) + }) + t.Run("probabilistic", func(t *testing.T) { if err := r.RunAgent([]byte("apm_config:\r\n probabilistic_sampler:\r\n enabled: true\r\n sampling_percentage: 100\r\n")); err != nil { t.Fatal(err) diff --git a/comp/README.md b/comp/README.md index 357b628c7a3c8..5c6b4887087f5 100644 --- a/comp/README.md +++ b/comp/README.md @@ -375,10 +375,6 @@ Package collector defines the OpenTelemetry Collector component. Package collectorcontrib defines the OTel collector-contrib component -### [comp/otelcol/configstore](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/otelcol/configstore) - -Package configstore defines the otel agent configstore component. - ### [comp/otelcol/converter](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/otelcol/converter) Package converter defines the otel agent converter component. diff --git a/comp/agent/bundle_test.go b/comp/agent/bundle_test.go index 7f60820804110..1f52881bc6596 100644 --- a/comp/agent/bundle_test.go +++ b/comp/agent/bundle_test.go @@ -8,8 +8,6 @@ package agent import ( "testing" - "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/agent/jmxlogger/jmxloggerimpl" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" @@ -30,8 +28,7 @@ func TestBundleDependencies(t *testing.T) { defaultforwarder.MockModule(), orchestratorimpl.MockModule(), eventplatformimpl.MockModule(), - demultiplexerimpl.Module(), - fx.Supply(demultiplexerimpl.NewDefaultParams()), + demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()), workloadmetafxmock.MockModule(workloadmeta.NewParams()), ) } diff --git a/comp/agent/jmxlogger/jmxloggerimpl/jmxlogger.go b/comp/agent/jmxlogger/jmxloggerimpl/jmxlogger.go index 9b0824b8fad41..27cd8485147fa 100644 --- a/comp/agent/jmxlogger/jmxloggerimpl/jmxlogger.go +++ b/comp/agent/jmxlogger/jmxloggerimpl/jmxlogger.go @@ -11,10 +11,10 @@ import ( "go.uber.org/fx" - "github.com/DataDog/datadog-agent/cmd/agent/common/path" "github.com/DataDog/datadog-agent/comp/agent/jmxlogger" "github.com/DataDog/datadog-agent/comp/core/config" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/log" pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup" @@ -53,7 +53,7 @@ func newJMXLogger(deps dependencies) (jmxlogger.Component, error) { syslogURI := pkglogsetup.GetSyslogURI(pkgconfigsetup.Datadog()) jmxLogFile := config.GetString("jmx_log_file") if jmxLogFile == "" { - jmxLogFile = path.DefaultJmxLogFile + jmxLogFile = defaultpaths.JmxLogFile } if config.GetBool("disable_file_logging") { diff --git a/comp/aggregator/bundle.go b/comp/aggregator/bundle.go index a818808a85fce..3539e731085cc 100644 --- a/comp/aggregator/bundle.go +++ b/comp/aggregator/bundle.go @@ -14,7 +14,7 @@ import ( // team: agent-metrics-logs // Bundle defines the fx options for this bundle. -func Bundle() fxutil.BundleOptions { +func Bundle(params demultiplexerimpl.Params) fxutil.BundleOptions { return fxutil.Bundle( - demultiplexerimpl.Module()) + demultiplexerimpl.Module(params)) } diff --git a/comp/aggregator/bundle_test.go b/comp/aggregator/bundle_test.go index ff208e7fb570f..d1a4cb2d4ecbd 100644 --- a/comp/aggregator/bundle_test.go +++ b/comp/aggregator/bundle_test.go @@ -8,8 +8,6 @@ package aggregator import ( "testing" - "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" @@ -20,12 +18,11 @@ import ( ) func TestBundleDependencies(t *testing.T) { - fxutil.TestBundle(t, Bundle(), + fxutil.TestBundle(t, Bundle(demultiplexerimpl.Params{}), core.MockBundle(), compressionimpl.MockModule(), defaultforwarder.MockModule(), orchestratorForwarderImpl.MockModule(), - fx.Supply(demultiplexerimpl.Params{}), eventplatformimpl.MockModule(), ) } diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go index d3b52d674ac8a..d51c825f847d1 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go @@ -13,6 +13,7 @@ import ( demultiplexerComp "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager" + "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" @@ -26,14 +27,16 @@ import ( ) // Module defines the fx options for this component. -func Module() fxutil.Module { +func Module(params Params) fxutil.Module { return fxutil.Component( - fx.Provide(newDemultiplexer)) + fx.Provide(newDemultiplexer), + fx.Supply(params)) } type dependencies struct { fx.In Lc fx.Lifecycle + Config config.Component Log log.Component SharedForwarder defaultforwarder.Component OrchestratorForwarder orchestratorforwarder.Component @@ -67,19 +70,19 @@ type provides struct { func newDemultiplexer(deps dependencies) (provides, error) { hostnameDetected, err := hostname.Get(context.TODO()) if err != nil { - if deps.Params.ContinueOnMissingHostname { + if deps.Params.continueOnMissingHostname { deps.Log.Warnf("Error getting hostname: %s", err) hostnameDetected = "" } else { return provides{}, deps.Log.Errorf("Error while getting hostname, exiting: %v", err) } } - + options := createAgentDemultiplexerOptions(deps.Config, deps.Params) agentDemultiplexer := aggregator.InitAndStartAgentDemultiplexer( deps.Log, deps.SharedForwarder, deps.OrchestratorForwarder, - deps.Params.AgentDemultiplexerOptions, + options, deps.EventPlatformForwarder, deps.Compressor, hostnameDetected) @@ -102,6 +105,19 @@ func newDemultiplexer(deps dependencies) (provides, error) { }, nil } +func createAgentDemultiplexerOptions(config config.Component, params Params) aggregator.AgentDemultiplexerOptions { + options := aggregator.DefaultAgentDemultiplexerOptions() + if params.useDogstatsdNoAggregationPipelineConfig { + options.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline") + } + + // Override FlushInterval only if flushInterval is set by the user + if v, ok := params.flushInterval.Get(); ok { + options.FlushInterval = v + } + return options +} + // LazyGetSenderManager gets an instance of SenderManager lazily. func (demux demultiplexer) LazyGetSenderManager() (sender.SenderManager, error) { return demux, nil diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/params.go b/comp/aggregator/demultiplexer/demultiplexerimpl/params.go index 1653ca820b7f5..b176d9b30a0ef 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/params.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/params.go @@ -5,18 +5,51 @@ package demultiplexerimpl -import "github.com/DataDog/datadog-agent/pkg/aggregator" +import ( + "time" + + "github.com/DataDog/datadog-agent/pkg/util/optional" +) // Params contains the parameters for the demultiplexer type Params struct { - aggregator.AgentDemultiplexerOptions - ContinueOnMissingHostname bool + continueOnMissingHostname bool + + // This is an optional field to override the default flush interval only if it is set + flushInterval optional.Option[time.Duration] + + useDogstatsdNoAggregationPipelineConfig bool } +// Option is a function that sets a parameter in the Params struct +type Option func(*Params) + // NewDefaultParams returns the default parameters for the demultiplexer -func NewDefaultParams() Params { - return Params{ - AgentDemultiplexerOptions: aggregator.DefaultAgentDemultiplexerOptions(), - ContinueOnMissingHostname: false, +func NewDefaultParams(options ...Option) Params { + p := Params{} + for _, o := range options { + o(&p) + } + return p +} + +// WithContinueOnMissingHostname sets the continueOnMissingHostname field to true +func WithContinueOnMissingHostname() Option { + return func(p *Params) { + p.continueOnMissingHostname = true + } +} + +// WithFlushInterval sets the flushInterval field to the provided duration +func WithFlushInterval(duration time.Duration) Option { + return func(p *Params) { + p.flushInterval = optional.NewOption(duration) + } +} + +// WithDogstatsdNoAggregationPipelineConfig uses the config dogstatsd_no_aggregation_pipeline +func WithDogstatsdNoAggregationPipelineConfig() Option { + return func(p *Params) { + p.useDogstatsdNoAggregationPipelineConfig = true } } diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go b/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go index 41fea8657e3d9..8ec83cb0cddf0 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/status_test.go @@ -61,7 +61,7 @@ func TestStatusOutPut(t *testing.T) { eventplatformimpl.MockModule(), fx.Supply( Params{ - ContinueOnMissingHostname: true, + continueOnMissingHostname: true, }, ), )) diff --git a/comp/api/api/apiimpl/grpc.go b/comp/api/api/apiimpl/grpc.go index 5ebe238c41547..099f56d9ae8c1 100644 --- a/comp/api/api/apiimpl/grpc.go +++ b/comp/api/api/apiimpl/grpc.go @@ -109,7 +109,7 @@ func (s *serverSecure) DogstatsdSetTaggerState(_ context.Context, req *pb.Tagger if taggerReplay == nil { return &pb.TaggerStateResponse{Loaded: false}, fmt.Errorf("unable to instantiate state") } - state := make([]taggerTypes.Entity, len(req.State)) + state := make([]taggerTypes.Entity, 0, len(req.State)) // better stores these as the native type for id, entity := range req.State { diff --git a/comp/api/api/def/go.mod b/comp/api/api/def/go.mod index e6856d630cfeb..3dc7234aeb00f 100644 --- a/comp/api/api/def/go.mod +++ b/comp/api/api/def/go.mod @@ -8,5 +8,5 @@ require ( go.uber.org/dig v1.18.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/sys v0.24.0 // indirect + golang.org/x/sys v0.25.0 // indirect ) diff --git a/comp/api/api/def/go.sum b/comp/api/api/def/go.sum index c26518bc2614a..e3997e581d5b4 100644 --- a/comp/api/api/def/go.sum +++ b/comp/api/api/def/go.sum @@ -14,7 +14,7 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/comp/api/authtoken/go.mod b/comp/api/authtoken/go.mod new file mode 100644 index 0000000000000..34abcae46bf7d --- /dev/null +++ b/comp/api/authtoken/go.mod @@ -0,0 +1,116 @@ +module github.com/DataDog/datadog-agent/comp/api/authtoken + +go 1.22.0 + +replace ( + github.com/DataDog/datadog-agent/comp/api/api/def => ../../../comp/api/api/def + github.com/DataDog/datadog-agent/comp/core/config => ../../../comp/core/config + github.com/DataDog/datadog-agent/comp/core/flare/builder => ../../../comp/core/flare/builder + github.com/DataDog/datadog-agent/comp/core/flare/types => ../../../comp/core/flare/types + github.com/DataDog/datadog-agent/comp/core/log/def => ../../../comp/core/log/def + github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../comp/core/log/mock + github.com/DataDog/datadog-agent/comp/core/secrets => ../../../comp/core/secrets + github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../comp/core/telemetry + github.com/DataDog/datadog-agent/comp/def => ../../../comp/def + github.com/DataDog/datadog-agent/pkg/api => ../../../pkg/api + github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../pkg/collector/check/defaults + github.com/DataDog/datadog-agent/pkg/config/env => ../../../pkg/config/env + github.com/DataDog/datadog-agent/pkg/config/mock => ../../../pkg/config/mock + github.com/DataDog/datadog-agent/pkg/config/model => ../../../pkg/config/model + github.com/DataDog/datadog-agent/pkg/config/nodetreemodel => ../../../pkg/config/nodetreemodel + github.com/DataDog/datadog-agent/pkg/config/setup => ../../../pkg/config/setup + github.com/DataDog/datadog-agent/pkg/config/teeconfig => ../../../pkg/config/teeconfig + github.com/DataDog/datadog-agent/pkg/config/utils => ../../../pkg/config/utils + github.com/DataDog/datadog-agent/pkg/util/defaultpaths => ../../../pkg/util/defaultpaths + github.com/DataDog/datadog-agent/pkg/util/executable => ../../../pkg/util/executable + github.com/DataDog/datadog-agent/pkg/util/filesystem => ../../../pkg/util/filesystem + github.com/DataDog/datadog-agent/pkg/util/fxutil => ../../../pkg/util/fxutil + github.com/DataDog/datadog-agent/pkg/util/hostname/validate => ../../../pkg/util/hostname/validate/ + github.com/DataDog/datadog-agent/pkg/util/log => ../../../pkg/util/log + github.com/DataDog/datadog-agent/pkg/util/log/setup => ../../../pkg/util/log/setup + github.com/DataDog/datadog-agent/pkg/util/optional => ../../../pkg/util/optional + github.com/DataDog/datadog-agent/pkg/util/pointer => ../../../pkg/util/pointer + github.com/DataDog/datadog-agent/pkg/util/scrubber => ../../../pkg/util/scrubber + github.com/DataDog/datadog-agent/pkg/util/system => ../../../pkg/util/system + github.com/DataDog/datadog-agent/pkg/util/system/socket => ../../../pkg/util/system/socket/ + github.com/DataDog/datadog-agent/pkg/util/testutil => ../../../pkg/util/testutil + github.com/DataDog/datadog-agent/pkg/util/winutil => ../../../pkg/util/winutil + github.com/DataDog/datadog-agent/pkg/version => ../../../pkg/version + +) + +require ( + github.com/DataDog/datadog-agent/comp/core/config v0.56.0 + github.com/DataDog/datadog-agent/comp/core/log/def v0.58.0-devel + github.com/DataDog/datadog-agent/comp/core/log/mock v0.58.0-devel + github.com/DataDog/datadog-agent/pkg/api v0.56.0 + github.com/DataDog/datadog-agent/pkg/util/fxutil v0.56.0 + github.com/DataDog/datadog-agent/pkg/util/optional v0.57.1 + github.com/stretchr/testify v1.9.0 + go.uber.org/fx v1.22.2 +) + +require ( + github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0 // indirect + github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0 // indirect + github.com/DataDog/datadog-agent/comp/core/secrets v0.57.1 // indirect + github.com/DataDog/datadog-agent/comp/def v0.56.0 // indirect + github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/config/env v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/config/mock v0.58.0-devel // indirect + github.com/DataDog/datadog-agent/pkg/config/model v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/config/nodetreemodel v0.60.0-devel // indirect + github.com/DataDog/datadog-agent/pkg/config/setup v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/config/teeconfig v0.60.0-devel // indirect + github.com/DataDog/datadog-agent/pkg/config/utils v0.56.0 // indirect + github.com/DataDog/datadog-agent/pkg/util/executable v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/filesystem v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/hostname/validate v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/log/setup v0.58.0-devel // indirect + github.com/DataDog/datadog-agent/pkg/util/pointer v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/system/socket v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/util/winutil v0.57.1 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.56.0 // indirect + github.com/DataDog/viper v1.13.5 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.1 // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/pelletier/go-toml v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/dig v1.18.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/tools v0.25.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/comp/api/authtoken/go.sum b/comp/api/authtoken/go.sum new file mode 100644 index 0000000000000..c4df7b42ad5d0 --- /dev/null +++ b/comp/api/authtoken/go.sum @@ -0,0 +1,357 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= +github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= +go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.22.2 h1:iPW+OPxv0G8w75OemJ1RAnTUrF55zOJlXlo1TbJ0Buw= +go.uber.org/fx v1.22.2/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190529164535-6a60838ec259/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/comp/autoscaling/datadogclient/impl/client.go b/comp/autoscaling/datadogclient/impl/client.go index 406b793b916ff..ee204f1f921f3 100644 --- a/comp/autoscaling/datadogclient/impl/client.go +++ b/comp/autoscaling/datadogclient/impl/client.go @@ -14,6 +14,7 @@ import ( configComponent "github.com/DataDog/datadog-agent/comp/core/config" logComp "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/status" + "github.com/DataDog/datadog-agent/pkg/config/structure" "gopkg.in/zorkian/go-datadog-api.v2" ) @@ -120,7 +121,7 @@ func (d *datadogClientWrapper) refreshClient() { func createDatadogClient(cfg configComponent.Component, logger logComp.Component) (datadogclient.Component, error) { if cfg.IsSet(metricsRedundantEndpointConfig) { var endpoints []endpoint - if err := cfg.UnmarshalKey(metricsRedundantEndpointConfig, &endpoints); err != nil { + if err := structure.UnmarshalKey(cfg, metricsRedundantEndpointConfig, &endpoints); err != nil { return nil, fmt.Errorf("could not parse %s: %v", metricsRedundantEndpointConfig, err) } diff --git a/comp/core/agenttelemetry/def/component.go b/comp/core/agenttelemetry/def/component.go index cb11fe507e305..9d02ba093213b 100644 --- a/comp/core/agenttelemetry/def/component.go +++ b/comp/core/agenttelemetry/def/component.go @@ -3,12 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// --------------------------------------------------- -// -// This is experimental code and is subject to change. -// -// --------------------------------------------------- - // Package agenttelemetry implements a component to generate Agent telemetry package agenttelemetry diff --git a/comp/core/agenttelemetry/impl/agenttelemetry.go b/comp/core/agenttelemetry/impl/agenttelemetry.go index 2b2881ed1f72d..3ca0c0e663268 100644 --- a/comp/core/agenttelemetry/impl/agenttelemetry.go +++ b/comp/core/agenttelemetry/impl/agenttelemetry.go @@ -3,30 +3,18 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// --------------------------------------------------- -// -// This is experimental code and is subject to change. -// -// --------------------------------------------------- - // Package agenttelemetryimpl provides the implementation of the agenttelemetry component. package agenttelemetryimpl import ( - "bytes" "context" - "embed" - "encoding/json" - "reflect" "strconv" - "strings" "golang.org/x/exp/maps" agenttelemetry "github.com/DataDog/datadog-agent/comp/core/agenttelemetry/def" "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" - "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/telemetry" compdef "github.com/DataDog/datadog-agent/comp/def" @@ -36,14 +24,10 @@ import ( // Embed one or more rendering templated into this binary as a resource // to be used at runtime. -//go:embed status_templates -var templatesFS embed.FS - type atel struct { - cfgComp config.Component - logComp log.Component - telComp telemetry.Component - statusComp status.Component + cfgComp config.Component + logComp log.Component + telComp telemetry.Component enabled bool sender sender @@ -61,7 +45,6 @@ type Requires struct { Log log.Component Config config.Component Telemetry telemetry.Component - Status status.Component Lifecycle compdef.Lifecycle } @@ -100,7 +83,6 @@ func createAtel( cfgComp config.Component, logComp log.Component, telComp telemetry.Component, - statusComp status.Component, sender sender, runner runner) *atel { // Parse Agent Telemetry Configuration configuration @@ -127,14 +109,13 @@ func createAtel( } return &atel{ - enabled: true, - cfgComp: cfgComp, - logComp: logComp, - telComp: telComp, - statusComp: statusComp, - sender: sender, - runner: runner, - atelCfg: atelCfg, + enabled: true, + cfgComp: cfgComp, + logComp: logComp, + telComp: telComp, + sender: sender, + runner: runner, + atelCfg: atelCfg, } } @@ -145,7 +126,6 @@ func NewComponent(req Requires) agenttelemetry.Component { req.Config, req.Log, req.Telemetry, - req.Status, nil, nil, ) @@ -348,160 +328,6 @@ func (a *atel) reportAgentMetrics(session *senderSession, p *Profile) { } } -// renderAgentStatus renders (transform) input status JSON object into output status using the template -func (a *atel) renderAgentStatus(p *Profile, inputStatus map[string]interface{}, statusOutput map[string]interface{}) { - // Render template if needed - if p.Status.Template == "none" { - return - } - - templateName := "agent-telemetry-" + p.Status.Template + ".tmpl" - var b = new(bytes.Buffer) - err := status.RenderText(templatesFS, templateName, b, inputStatus) - if err != nil { - a.logComp.Errorf("Failed to collect Agent Status telemetry. Error: %s", err.Error()) - return - } - if len(b.Bytes()) == 0 { - a.logComp.Debug("Agent status rendering to agent telemetry payloads return empty payload") - return - } - - // Convert byte slice to JSON object - if err := json.Unmarshal(b.Bytes(), &statusOutput); err != nil { - a.logComp.Errorf("Failed to collect Agent Status telemetry. Error: %s", err.Error()) - return - } -} - -func (a *atel) addAgentStatusExtra(p *Profile, fullStatus map[string]interface{}, statusOutput map[string]interface{}) { - for _, builder := range p.statusExtraBuilder { - // Evaluate JQ expression against the agent status JSON object - jqResult := builder.jqSource.Run(fullStatus) - jqValue, ok := jqResult.Next() - if !ok { - a.logComp.Errorf("Failed to apply JQ expression for JSON path '%s' to Agent Status payload. Error unknown", - strings.Join(builder.jpathTarget, ".")) - continue - } - - // Validate JQ expression result - if err, ok := jqValue.(error); ok { - a.logComp.Errorf("Failed to apply JQ expression for JSON path '%s' to Agent Status payload. Error: %s", - strings.Join(builder.jpathTarget, "."), err.Error()) - continue - } - - // Validate JQ expression result type - var attrVal interface{} - switch jqValueType := jqValue.(type) { - case int: - attrVal = jqValueType - case float64: - attrVal = jqValueType - case bool: - attrVal = jqValueType - case nil: - a.logComp.Errorf("JQ expression return 'nil' value for JSON path '%s'", strings.Join(builder.jpathTarget, ".")) - continue - case string: - a.logComp.Errorf("string value (%v) for JSON path '%s' for extra status atttribute is not currently allowed", - strings.Join(builder.jpathTarget, "."), attrVal) - continue - default: - a.logComp.Errorf("'%v' value (%v) for JSON path '%s' for extra status atttribute is not currently allowed", - reflect.TypeOf(jqValueType), reflect.ValueOf(jqValueType), strings.Join(builder.jpathTarget, ".")) - continue - } - - // Add resulting value to the agent status telemetry payload (recursively creating missing JSON objects) - curNode := statusOutput - for i, p := range builder.jpathTarget { - // last element is the attribute name - if i == len(builder.jpathTarget)-1 { - curNode[p] = attrVal - break - } - - existSubNode, ok := curNode[p] - - // if the node doesn't exist, create it - if !ok { - newSubNode := make(map[string]interface{}) - curNode[p] = newSubNode - curNode = newSubNode - } else { - existSubNodeCasted, ok := existSubNode.(map[string]interface{}) - if !ok { - a.logComp.Errorf("JSON path '%s' points to non-object element", strings.Join(builder.jpathTarget[:i], ".")) - break - } - curNode = existSubNodeCasted - } - } - } -} - -// Render Agent Status JSON object (using template if needed and adding extra attributes if specified in -// the profile). The rendered JSON object is then sent to the telemetry server. The "rendering" of the -// Agent Status JSON object may appear odd and flawed. For example ... -// - Agent Status, generally speaking, is moving into direction when multitudes of its providers are -// starting self-"rendering". Accordingly, its JSON representation is not fixed and certainly -// not-public and is subject to change, which may break both template and JQ style of rendering -// of Agent Telemetry (and which is implemented in this function). It is certainly a concern but -// its potential impact is minimized since Agent Telemetry is an internal feature. It is also -// the price of flexibility and decoupling rendering from the code. -// - There are a number of inefficiencies in the current implementation massaging Agent Status JSON -// object into JSON, then to bytes and JSON again. It should not be a big issue since the operations -// will be very infrequent and resulting objects relatively small. -// - In some way, the current implementation can be thought of as an architectural shortcut since -// Agent Status, arguably is not purely telemetry data (however it can be argue that it is also can -// be think of as quintessential Agent telemetry data) and perhaps more explicit Agent telemetry -// interfaces should be implemented where its loosely coupled "providers" supply their own internal -// telemetry to be emitted. -func (a *atel) reportAgentStatus(session *senderSession, p *Profile) { - // If no status is configured nothing to report - if p.Status == nil { - return - } - - a.logComp.Debugf("Collect Agent Status telemetry for profile %s", p.Name) - - // Current "agent-telemetry-basic.tmpl" uses only "runneStats" and "dogstatsdStats" JSON sections - // These JSON sections are populated via "collector" and "DogStatsD" status providers sections - minimumReqSections := []string{"collector", "DogStatsD"} - statusBytes, err := a.statusComp.GetStatusBySections(minimumReqSections, "json", false) - if err != nil { - a.logComp.Errorf("failed to get agent status: %s", err) - return - } - - var statusJSON = make(map[string]interface{}) - err = json.Unmarshal(statusBytes, &statusJSON) - if err != nil { - a.logComp.Errorf("failed to unmarshall agent status: %s", err) - return - } - - // Render Agent Status JSON object (using template if needed and adding extra attributes) - var statusPayloadJSON = make(map[string]interface{}) - a.renderAgentStatus(p, statusJSON, statusPayloadJSON) - a.addAgentStatusExtra(p, statusJSON, statusPayloadJSON) - if len(statusPayloadJSON) == 0 { - a.logComp.Debug("No Agent Status telemetry collected") - return - } - - a.logComp.Debugf("Reporting Agent Status telemetry for profile %s", p.Name) - - // Send the Agent Telemetry status payload - err = a.sender.sendAgentStatusPayload(session, statusPayloadJSON) - if err != nil { - a.logComp.Errorf("failed to send agent status: %s", err) - return - } -} - // run runs the agent telemetry for a given profile. It is triggered by the runner // according to the profiles schedule. func (a *atel) run(profiles []*Profile) { @@ -516,7 +342,6 @@ func (a *atel) run(profiles []*Profile) { for _, p := range profiles { a.reportAgentMetrics(session, p) - a.reportAgentStatus(session, p) } err := a.sender.flushSession(session) @@ -533,7 +358,7 @@ func (a *atel) GetAsJSON() ([]byte, error) { // start is called by FX when the application starts. func (a *atel) start() error { - a.logComp.Info("Starting agent telemetry for %d schedules and %d profiles", len(a.atelCfg.schedule), len(a.atelCfg.Profiles)) + a.logComp.Infof("Starting agent telemetry for %d schedules and %d profiles", len(a.atelCfg.schedule), len(a.atelCfg.Profiles)) a.cancelCtx, a.cancel = context.WithCancel(context.Background()) diff --git a/comp/core/agenttelemetry/impl/agenttelemetry_test.go b/comp/core/agenttelemetry/impl/agenttelemetry_test.go index a47686d674389..a1cdfe85409a9 100644 --- a/comp/core/agenttelemetry/impl/agenttelemetry_test.go +++ b/comp/core/agenttelemetry/impl/agenttelemetry_test.go @@ -3,12 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// --------------------------------------------------- -// -// This is experimental code and is subject to change. -// -// --------------------------------------------------- - package agenttelemetryimpl import ( @@ -28,8 +22,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" - "github.com/DataDog/datadog-agent/comp/core/status" - "github.com/DataDog/datadog-agent/comp/core/status/statusimpl" "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -63,9 +55,6 @@ func (s *senderMock) startSession(_ context.Context) *senderSession { func (s *senderMock) flushSession(_ *senderSession) error { return nil } -func (s *senderMock) sendAgentStatusPayload(_ *senderSession, _ map[string]interface{}) error { - return nil -} func (s *senderMock) sendAgentMetricPayloads(_ *senderSession, metrics []*agentmetric) error { s.sentMetrics = append(s.sentMetrics, metrics...) return nil @@ -147,10 +136,6 @@ func makeLogMock(t *testing.T) log.Component { return logmock.New(t) } -func makeStatusMock(t *testing.T) status.Component { - return fxutil.Test[status.Mock](t, fx.Options(statusimpl.MockModule())) -} - func makeSenderImpl(t *testing.T, c string) sender { o := convertYamlStrToMap(t, c) cfg := makeCfgMock(t, o) @@ -190,7 +175,7 @@ func getTestAtel(t *testing.T, } assert.NoError(t, err) - atel := createAtel(cfg, log, tel, makeStatusMock(t), sndr, runner) + atel := createAtel(cfg, log, tel, sndr, runner) if atel == nil { err = fmt.Errorf("failed to create atel") } @@ -257,12 +242,6 @@ func TestEnableIfNotGovCloud(t *testing.T) { assert.True(t, a.enabled) } -func TestDisableByDefault(t *testing.T) { - o := map[string]any{"foo": "bar", "site": "foo.bar"} - a := getTestAtel(t, nil, o, nil, nil, nil) - assert.False(t, a.enabled) -} - func TestRun(t *testing.T) { r := newRunnerMock() o := getCommonOverrideConfig(true, "foo.bar") @@ -271,10 +250,10 @@ func TestRun(t *testing.T) { a.start() - // default configuration has 1 job with 2 profiles (more configurations needs to be tested) + // default configuration has 2 job with 2 profiles (more configurations needs to be tested) // will be improved in future by providing deterministic configuration - assert.Equal(t, 1, len(r.(*runnerMock).jobs)) - assert.Equal(t, 2, len(r.(*runnerMock).jobs[0].profiles)) + assert.Equal(t, 2, len(r.(*runnerMock).jobs)) + assert.Equal(t, 1, len(r.(*runnerMock).jobs[0].profiles)) } func TestReportMetricBasic(t *testing.T) { diff --git a/comp/core/agenttelemetry/impl/config.go b/comp/core/agenttelemetry/impl/config.go index 3109a400b573c..f04ca5c10437b 100644 --- a/comp/core/agenttelemetry/impl/config.go +++ b/comp/core/agenttelemetry/impl/config.go @@ -3,12 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// --------------------------------------------------- -// -// This is experimental code and is subject to change. -// -// --------------------------------------------------- - package agenttelemetryimpl import ( @@ -40,14 +34,12 @@ type Profile struct { // parsed Name string `yaml:"name"` Metric *AgentMetricConfig `yaml:"metric,omitempty"` - Status *AgentStatusConfig `yaml:"status"` Schedule *Schedule `yaml:"schedule"` // compiled - statusExtraBuilder []jBuilder - metricsMap map[string]*MetricConfig - excludeZeroMetric bool - excludeTagsMap map[string]any + metricsMap map[string]*MetricConfig + excludeZeroMetric bool + excludeTagsMap map[string]any } // AgentMetricConfig specifies agent telemetry metrics payloads to be generated and emitted @@ -73,12 +65,6 @@ type MetricConfig struct { aggregateTagsMap map[string]any } -// AgentStatusConfig is a single agent telemetry status payload -type AgentStatusConfig struct { - Template string `yaml:"template"` - Extra map[string]string `yaml:"extra,omitempty"` -} - // Schedule is a schedule for agent telemetry payloads to be generated and emitted type Schedule struct { // parsed @@ -135,63 +121,6 @@ type Schedule struct { // reserved tag"). If not specified, specified, default value of `false` will be used. // It is useful only if "aggregate_tags" is also specified and will be ignored otherwise. // -// profiles[].status (optional) -// -------------------------------- -// When included, agent telemetry status payloads will be generated and emitted. -// -// profiles[].status.template (optional) -// -------------------------------------- -// Name of agent status JSON rendering template which generates agent telemetry status -// payload. Used as a suffix to -// "pkg\status\render\templates\agent-telemetry-