From d93bf87be99163a70c49e2ec416d7434f2bf0371 Mon Sep 17 00:00:00 2001
From: Chris Berkhout <chris.berkhout@elastic.co>
Date: Tue, 7 Jan 2025 14:24:20 +0100
Subject: [PATCH] [jamf_pro] Inventory pagination fix (#12231)

* Always include the General section of inventory data.
* Update the cursor whenever there is new data.
* Assume there isn't another page if the current page isn't full.
* Correct the name of the `enable_section_content_caching` variable.
* Adjust the `page_size` variable description to correct the maximum.
* Simplify code to set the inventory filter parameter.

This has been manually tested against the live API.
---
 packages/jamf_pro/changelog.yml               |  5 ++++
 .../inventory/agent/stream/cel.yml.hbs        | 23 ++++++++-----------
 .../data_stream/inventory/manifest.yml        | 20 +++++++---------
 packages/jamf_pro/manifest.yml                |  2 +-
 4 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/packages/jamf_pro/changelog.yml b/packages/jamf_pro/changelog.yml
index 5795982a0b9..02499167da6 100644
--- a/packages/jamf_pro/changelog.yml
+++ b/packages/jamf_pro/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.1"
+  changes:
+    - description: Inventory pagination fix.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12231
 - version: "0.2.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
diff --git a/packages/jamf_pro/data_stream/inventory/agent/stream/cel.yml.hbs b/packages/jamf_pro/data_stream/inventory/agent/stream/cel.yml.hbs
index e4620184afe..2bd018a2c61 100644
--- a/packages/jamf_pro/data_stream/inventory/agent/stream/cel.yml.hbs
+++ b/packages/jamf_pro/data_stream/inventory/agent/stream/cel.yml.hbs
@@ -37,10 +37,7 @@ program: |-
   		"section": state.sections,
   		"page-size": [string(state.page_size)],
   		"sort": ["general.reportDate:asc"],
-  		?"filter": (has(state.?cursor.last_report_date) && state.?cursor.last_report_date.orValue("") != "") ?
-  			optional.of(["general.reportDate>=\"" + state.cursor.last_report_date + "\""])
-  		:
-  			optional.none(),
+  		?"filter": state.?cursor.last_report_date.optMap(d, ["general.reportDate>=\"" + d + "\""]),
   	}.format_query()
   ).with(
   	{
@@ -67,13 +64,15 @@ program: |-
   						?"event.original": state.?preserve_original_event.orValue(false) ? optional.of(e.encode_json()) : optional.none(),
   					}
   				),
-  				"want_more": body.totalCount > size(body.results),
-  				"cursor": {
-  					"last_report_date": (size(body.results) == 0 || size(body.results) == body.totalCount) ?
-  						state.?cursor.last_report_date.orValue("")
-  					:
-  						string(body.results[size(body.results) - 1].general.reportDate),
-  				},
+  				"want_more": size(body.results) >= state.page_size,
+  				"cursor": state.?cursor.orValue({}).with(
+  					{
+  						?"last_report_date": (size(body.results) > 0) ?
+  							optional.of(body.results[size(body.results) - 1].general.reportDate)
+  						:
+  							optional.none(),
+  					}
+  				),
   			}
   		)
   	)
@@ -82,9 +81,7 @@ state:
   preserve_original_event: {{preserve_original_event}}
   page_size: {{page_size}}
   sections:
-  {{#if enable_section_general }}
     - GENERAL
-  {{/if}}
   {{#if enable_section_hardware }}
     - HARDWARE
   {{/if}}
diff --git a/packages/jamf_pro/data_stream/inventory/manifest.yml b/packages/jamf_pro/data_stream/inventory/manifest.yml
index b9bc333beaa..d2c77550cf2 100644
--- a/packages/jamf_pro/data_stream/inventory/manifest.yml
+++ b/packages/jamf_pro/data_stream/inventory/manifest.yml
@@ -1,12 +1,14 @@
-title: "Inventory data"
+title: Inventory data
 type: logs
 streams:
   - input: cel
     template_path: cel.yml.hbs
-    title: inventory_data
+    title: Inventory data
     description: |
-      Inventory requests settings
-      "*NOTE* Many sections enabled may impact the query processing. It is not recommended to enable setions not required"
+      Settings for inventory requests.  
+      *NOTE* Enabling many sections may impact query processing.
+      It is recommended to only enable sections of interest.
+      The General section is always included.
     vars:
       - name: interval
         type: text
@@ -20,7 +22,7 @@ streams:
         required: false
         title: Inventory request page size
         show_user: false
-        description: Set page size for computers-inventory API Call. 100 max, should be adjusted if size of request is growing to reduce parsing time
+        description: Set page size for computers-inventory API Call. Must not exceed 1000.
         default: 100
       - name: preserve_original_event
         type: bool
@@ -46,12 +48,6 @@ streams:
         show_user: false
         default:
           - forwarded
-      - name: enable_section_general
-        type: bool
-        show_user: false
-        default: true
-        title: "General"
-        description: "Enable General Settings for the API responce"
       - name: enable_section_hardware
         type: bool
         show_user: false
@@ -178,7 +174,7 @@ streams:
         default: false
         title: "Extension Attributes"
         description: "Enable block with Extension Attributes"
-      - name: CONTENT_CACHING
+      - name: enable_section_content_caching
         type: bool
         show_user: false
         default: false
diff --git a/packages/jamf_pro/manifest.yml b/packages/jamf_pro/manifest.yml
index 914c2829fd2..abd8b03c5a1 100644
--- a/packages/jamf_pro/manifest.yml
+++ b/packages/jamf_pro/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.1.5
 name: jamf_pro
 title: "Jamf Pro"
-version: 0.2.0
+version: 0.2.1
 source:
   license: "Elastic-2.0"
 description: "Collect logs and inventory data from Jamf Pro with Elastic Agent"