diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 038c042ced..080fa46463 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -397,7 +397,13 @@ export function SetupBottomBar({ '{table_name}', `${config.connectionDataSource}.default.${config.connectionTableName}` ); - queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation); + // We append to this URI in internal queries, so we normalize it to have no trailing slash + let trimmedLocation = config.connectionLocation.trim(); + trimmedLocation = trimmedLocation.endsWith('/') + ? trimmedLocation.slice(0, trimmedLocation.length - 1) + : trimmedLocation; + + queryStr = queryStr.replaceAll('{s3_bucket_location}', trimmedLocation); queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName); queryStr = queryStr.replaceAll(/\s+/g, ' '); const result = await runQuery(queryStr, config.connectionDataSource, sessionId); diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql index ecb48c084e..86a23f32f9 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql @@ -38,4 +38,10 @@ SELECT classification as `aws.elb.classification`, classification_reason as `aws.elb.classification_reason` FROM - {table_name}; + {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql deleted file mode 100644 index 0f6ba98960..0000000000 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 27480a5d07..6fbc6714fe 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -61,11 +61,6 @@ "name": "create_mv", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv", - "version": "1.0.0", - "language": "sql" } ] }, diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql index 2612638e74..0706db3b5a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql @@ -21,4 +21,10 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS ELSE 'egress' END AS `aws.vpc.flow-direction` FROM - {table_name}; + {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql deleted file mode 100644 index 458202eaeb..0000000000 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 2a2fbf3f56..6720e0be93 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -57,11 +57,6 @@ "name": "create_mv_vpc", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv_vpc", - "version": "1.0.0", - "language": "sql" } ] }, diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql index 8b29fdf0b0..cb36863843 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql @@ -8,3 +8,9 @@ SELECT body_bytes_sent AS `http.response.bytes`, 'nginx.access' AS `event.domain` FROM {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql deleted file mode 100644 index 458202eaeb..0000000000 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index aadcae2f04..ec821ecf0c 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -49,11 +49,6 @@ "name": "create_mv", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv", - "version": "1.0.0", - "language": "sql" } ] },