Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Flint queries for Apache Access integration #1681

Merged
merged 11 commits into from
Apr 15, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,23 @@
"description": "Insights on applications using the standard Apache access log format.",
"license": "Apache-2.0",
"type": "logs_apache",
"labels": ["Observability", "Logs"],
"labels": ["Observability", "Logs", "Flint S3"],
"author": "OpenSearch",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info",
"workflows": [
{
"name": "queries",
"label": "Queries (recommended)",
"description": "Tables and pre-written queries for quickly getting insights on your data.",
"enabled_by_default": true
},
{
"name": "dashboards",
"label": "Dashboards & Visualizations",
"description": "Dashboards and indices that enable you to easily visualize important metrics.",
"enabled_by_default": false
}
],
"statics": {
"logo": {
"annotation": "Apache Logo",
Expand Down Expand Up @@ -39,7 +53,21 @@
"name": "apache",
"version": "1.0.0",
"extension": "ndjson",
"type": "savedObjectBundle"
"type": "savedObjectBundle",
"workflows": ["dashboards"]
},
{
"name": "create_table",
"version": "1.0.0",
"extension": "sql",
"type": "query"
},
{
"name": "create_mv",
"version": "1.0.0",
"extension": "sql",
"type": "query",
"workflows": ["dashboards"]
}
],
"sampleData": {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
CREATE MATERIALIZED VIEW {table_name}_mview AS
SELECT
to_timestamp(trim(BOTH '[]' FROM concat(time_local_1, ' ', time_local_2)), 'dd/MMM/yyyy:HH:mm:ss Z') AS `@timestamp`,
split_part (request, ' ', 1) as `http.request.method`,
split_part (request, ' ', 2) as `http.url`,
split_part (request, ' ', 3) as `http.flavor`,
status AS `http.response.status_code`,
body_bytes_sent AS `http.response.bytes`,
'apache.access' AS `event.domain`
FROM {table_name}
WITH (
auto_refresh = 'true',
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
remote_addr STRING,
empty_col STRING,
remote_user STRING,
time_local_1 STRING,
time_local_2 STRING,
request STRING,
status INT,
body_bytes_sent INT,
http_referer STRING,
http_user_agent STRING
) USING csv
Swiddis marked this conversation as resolved.
Show resolved Hide resolved
LOCATION '{s3_bucket_location}'
OPTIONS (
sep=' ',
nullValue='-'
)
Loading