Skip to content

Commit

Permalink
Add initial S3 integration
Browse files Browse the repository at this point in the history
Signed-off-by: Simeon Widdis <sawiddis@amazon.com>
  • Loading branch information
Swiddis committed Nov 1, 2023
1 parent 4c93be7 commit ae12ecf
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 2 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
CREATE MATERIALIZED VIEW {table_name}_mview AS
SELECT
to_timestamp(trim(BOTH '[]' FROM concat(request_datetime_1, ' ', request_datetime_2)), 'dd/MMM/yyyy:HH:mm:ss Z') as `@timestamp`,
bucket_name as `aws.s3.bucket`,
remote_ip as `communication.source.ip`,
remote_ip as `aws.s3.remote_ip`,
request_id as `aws.s3.request_id`,
operation as `aws.s3.operation`,
request_key as `aws.s3.key`,
request_uri as `aws.s3.request_uri`,
http_status as `http.response.status_code`,
http_status as `aws.s3.http_status`,
error_code as `aws.s3.error_code`,
bytes_sent as `aws.s3.bytes_sent`,
object_size as `aws.s3.object_size`,
total_time as `aws.s3.total_time`,
turn_around_time as `aws.s3.turn_around_time`,
referrer as `http.referrer`,
user_agent as `http.user_agent.original`,
version_id as `aws.s3.version_id`,
host_id as `aws.s3.host_id`,
signature_version as `aws.s3.signature_version`,
cipher_suite as `aws.s3.cipher_suite`,
auth_type as `aws.s3.authentication_type`,
host_header as `aws.s3.host_header`,
tls_version as `aws.s3.tls_version`
FROM
{table_name}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
bucket_owner STRING,
bucket_name STRING,
request_datetime_1 STRING,
request_datetime_2 STRING,
remote_ip STRING,
request_arn STRING,
request_id STRING,
operation STRING,
request_key STRING,
request_uri STRING,
http_status INT,
error_code STRING,
bytes_sent INT,
object_size INT,
total_time INT,
turn_around_time INT,
referrer STRING,
user_agent STRING,
version_id STRING,
host_id STRING,
signature_version STRING,
cipher_suite STRING,
auth_type STRING,
host_header STRING,
tls_version STRING
) USING csv LOCATION '{s3_bucket_location}' OPTIONS (sep = ' ')
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
REFRESH MATERIALIZED VIEW {table_name}_mview
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"description": "AWS S3 Object Store",
"license": "Apache-2.0",
"type": "logs_s3",
"labels": ["Observability", "Logs", "AWS", "Cloud"],
"labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
"author": "OpenSearch",
"sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info",
"statics": {
Expand Down Expand Up @@ -38,7 +38,24 @@
"savedObjects": {
"name": "aws_s3",
"version": "1.0.0"
}
},
"queries": [
{
"name": "create_table",
"version": "1.0.0",
"language": "sql"
},
{
"name": "create_mv",
"version": "1.0.0",
"language": "sql"
},
{
"name": "refresh_mv",
"version": "1.0.0",
"language": "sql"
}
]
},
"sampleData": {
"path": "sample.json"
Expand Down

0 comments on commit ae12ecf

Please sign in to comment.