Add Flint queries for Apache Access integration (opensearch-project#1681

)

* Add nginx integration

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Add VPC S3 integration

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Add basic apache integration based on nginx

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Add initial S3 integration

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Apply keyword changes to s3

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Remove old S3 attempt

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Adapt apache query loading to new config format

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

* Remove old aws s3 attempt (again)

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>

---------

Signed-off-by: Simeon Widdis <sawiddis@amazon.com>
(cherry picked from commit bbcd3c1)

server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json

@@ -5,9 +5,23 @@
   "description": "Insights on applications using the standard Apache access log format.",
   "license": "Apache-2.0",
   "type": "logs_apache",
-  "labels": ["Observability", "Logs"],
+  "labels": ["Observability", "Logs", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "Apache Logo",
@@ -39,7 +53,21 @@
       "name": "apache",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
+    },
+    {
+      "name": "create_table",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query"
+    },
+    {
+      "name": "create_mv",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {

server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql

@@ -0,0 +1,16 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+SELECT
+    to_timestamp(trim(BOTH '[]' FROM concat(time_local_1, ' ', time_local_2)), 'dd/MMM/yyyy:HH:mm:ss Z') AS `@timestamp`,
+    split_part (request, ' ', 1) as `http.request.method`,
+    split_part (request, ' ', 2) as `http.url`,
+    split_part (request, ' ', 3) as `http.flavor`,
+    status AS `http.response.status_code`,
+    body_bytes_sent AS `http.response.bytes`,
+    'apache.access' AS `event.domain`
+FROM {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);

server/adaptors/integrations/__data__/repository/apache/assets/create_table-1.0.0.sql

@@ -0,0 +1,17 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+  remote_addr STRING,
+  empty_col STRING,
+  remote_user STRING,
+  time_local_1 STRING,
+  time_local_2 STRING,
+  request STRING,
+  status INT,
+  body_bytes_sent INT,
+  http_referer STRING,
+  http_user_agent STRING
+) USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+  sep=' ',
+  nullValue='-'
+)