Cloudflare Logpush: Add request tracing support (#12224)

Add request trace logging to the Cloudflare Logpush integration.
elastic · Jan 7, 2025 · 191eb51 · 191eb51
1 parent 38db94e
commit 191eb51
Show file tree

Hide file tree

Showing 56 changed files with 240 additions and 1 deletion.
diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.0"
+  changes:
+    - description: Provide request tracing support.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12224
 - version: "1.29.0"
   changes:
     - description: Add support for Access Point ARN when collecting logs via the AWS S3 Bucket.

diff --git a/...udflare_logpush/data_stream/access_request/_dev/test/system/test-http-endpoint-config.yml b/...udflare_logpush/data_stream/access_request/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/access_request/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/access_request/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/access_request/manifest.yml b/packages/cloudflare_logpush/data_stream/access_request/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Access Request logs
     description: Collect Access Request logs from Cloudflare via S3 or SQS.

diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/test/system/test-http-endpoint-config.yml b/packages/cloudflare_logpush/data_stream/audit/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/audit/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/audit/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/audit/manifest.yml b/packages/cloudflare_logpush/data_stream/audit/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Audit logs
     description: Collect Audit logs from Cloudflare via S3 or SQS.

diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/test/system/test-http-endpoint-config.yml b/packages/cloudflare_logpush/data_stream/casb/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/casb/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/casb/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/casb/manifest.yml b/packages/cloudflare_logpush/data_stream/casb/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: CASB Findings logs
     description: Collect CASB Findings logs from Cloudflare via S3 or SQS.

diff --git a/...udflare_logpush/data_stream/device_posture/_dev/test/system/test-http-endpoint-config.yml b/...udflare_logpush/data_stream/device_posture/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/device_posture/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/device_posture/manifest.yml b/packages/cloudflare_logpush/data_stream/device_posture/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Device Posture Results logs
     description: Collect Device Posture Results logs from Cloudflare via S3 or SQS.

diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/test/system/test-http-endpoint-config.yml b/packages/cloudflare_logpush/data_stream/dns/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/dns/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/dns/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/dns/manifest.yml b/packages/cloudflare_logpush/data_stream/dns/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: DNS logs
     description: Collect DNS logs from Cloudflare via S3 or SQS.

diff --git a/...loudflare_logpush/data_stream/dns_firewall/_dev/test/system/test-http-endpoint-config.yml b/...loudflare_logpush/data_stream/dns_firewall/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/dns_firewall/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/manifest.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: DNS Firewall logs
     description: Collect DNS Firewall logs from Cloudflare via S3 or SQS.

diff --git a/...udflare_logpush/data_stream/firewall_event/_dev/test/system/test-http-endpoint-config.yml b/...udflare_logpush/data_stream/firewall_event/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/firewall_event/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/manifest.yml b/packages/cloudflare_logpush/data_stream/firewall_event/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Firewall Event logs
     description: Collect Firewall Event logs from Cloudflare via S3 or SQS.

diff --git a/...cloudflare_logpush/data_stream/gateway_dns/_dev/test/system/test-http-endpoint-config.yml b/...cloudflare_logpush/data_stream/gateway_dns/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_dns/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Gateway DNS logs
     description: Collect Gateway DNS logs from Cloudflare via S3 or SQS.

diff --git a/...loudflare_logpush/data_stream/gateway_http/_dev/test/system/test-http-endpoint-config.yml b/...loudflare_logpush/data_stream/gateway_http/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_http/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_http/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Gateway HTTP logs
     description: Collect Gateway HTTP logs from Cloudflare via S3 or SQS.

diff --git a/...dflare_logpush/data_stream/gateway_network/_dev/test/system/test-http-endpoint-config.yml b/...dflare_logpush/data_stream/gateway_network/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/gateway_network/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_network/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Gateway Network logs
     description: Collect Gateway Network logs from Cloudflare via S3 or SQS.

diff --git a/...loudflare_logpush/data_stream/http_request/_dev/test/system/test-http-endpoint-config.yml b/...loudflare_logpush/data_stream/http_request/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/http_request/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/http_request/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/http_request/manifest.yml b/packages/cloudflare_logpush/data_stream/http_request/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: HTTP Request logs
     description: Collect HTTP Request logs from Cloudflare via S3 or SQS.

diff --git a/...s/cloudflare_logpush/data_stream/magic_ids/_dev/test/system/test-http-endpoint-config.yml b/...s/cloudflare_logpush/data_stream/magic_ids/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/magic_ids/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml b/packages/cloudflare_logpush/data_stream/magic_ids/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: Magic IDS logs
     description: Collect Magic IDS logs from Cloudflare via S3 or SQS.

diff --git a/.../cloudflare_logpush/data_stream/nel_report/_dev/test/system/test-http-endpoint-config.yml b/.../cloudflare_logpush/data_stream/nel_report/_dev/test/system/test-http-endpoint-config.yml
@@ -9,3 +9,4 @@ data_stream:
     listen_port: 9560
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    enable_request_tracer: true
diff --git a/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/http_endpoint.yml.hbs b/packages/cloudflare_logpush/data_stream/nel_report/agent/stream/http_endpoint.yml.hbs
@@ -38,4 +38,8 @@ ssl: {{ssl}}
 {{#if processors}}
 processors:
 {{processors}}
+{{/if}}
+{{#if enable_request_tracer}}
+tracer.filename: "../../logs/http_endpoint/http-request-trace-*.ndjson"
+tracer.maxbackups: 5
 {{/if}}
diff --git a/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml b/packages/cloudflare_logpush/data_stream/nel_report/manifest.yml
@@ -70,6 +70,14 @@ streams:
         required: false
         show_user: false
         default: 10
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.
   - input: aws-s3
     title: NEL Report logs
     description: Collect NEL Report logs from Cloudflare via S3 or SQS.