diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql new file mode 100644 index 000000000..fd83cb708 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql @@ -0,0 +1,28 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + to_timestamp(trim(BOTH '[]' FROM concat(request_datetime_1, ' ', request_datetime_2)), 'dd/MMM/yyyy:HH:mm:ss Z') as `@timestamp`, + bucket_name as `aws.s3.bucket`, + remote_ip as `communication.source.ip`, + remote_ip as `aws.s3.remote_ip`, + request_id as `aws.s3.request_id`, + operation as `aws.s3.operation`, + request_key as `aws.s3.key`, + request_uri as `aws.s3.request_uri`, + http_status as `http.response.status_code`, + http_status as `aws.s3.http_status`, + error_code as `aws.s3.error_code`, + bytes_sent as `aws.s3.bytes_sent`, + object_size as `aws.s3.object_size`, + total_time as `aws.s3.total_time`, + turn_around_time as `aws.s3.turn_around_time`, + referrer as `http.referrer`, + user_agent as `http.user_agent.original`, + version_id as `aws.s3.version_id`, + host_id as `aws.s3.host_id`, + signature_version as `aws.s3.signature_version`, + cipher_suite as `aws.s3.cipher_suite`, + auth_type as `aws.s3.authentication_type`, + host_header as `aws.s3.host_header`, + tls_version as `aws.s3.tls_version` +FROM + {table_name} \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql new file mode 100644 index 000000000..d76580a9e --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql @@ -0,0 +1,27 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + bucket_owner STRING, + bucket_name STRING, + request_datetime_1 STRING, + request_datetime_2 STRING, + remote_ip STRING, + request_arn STRING, + request_id STRING, + operation STRING, + request_key STRING, + request_uri STRING, + http_status INT, + error_code STRING, + bytes_sent INT, + object_size INT, + total_time INT, + turn_around_time INT, + referrer STRING, + user_agent STRING, + version_id STRING, + host_id STRING, + signature_version STRING, + cipher_suite STRING, + auth_type STRING, + host_header STRING, + tls_version STRING +) USING csv LOCATION '{s3_bucket_location}' OPTIONS (sep = ' ') \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/refresh_mv-1.0.0.sql new file mode 100644 index 000000000..53ce88552 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/refresh_mv-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json index c519e57b7..bcf55b9c6 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS S3 Object Store", "license": "Apache-2.0", "type": "logs_s3", - "labels": ["Observability", "Logs", "AWS", "Cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", "statics": { @@ -38,7 +38,24 @@ "savedObjects": { "name": "aws_s3", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "create_table", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "create_mv", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "refresh_mv", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json"