Merge pull request #93 from rudderlabs/sync.v0.44.1

Sync.v0.44.1

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.29
+current_version = 0.42.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -8,62 +8,16 @@ serialize =
 
 [bumpversion:file:.bumpversion.cfg]
 
-[bumpversion:file:.env]
-
-[bumpversion:file:airbyte-bootloader/Dockerfile]
-
-[bumpversion:file:airbyte-container-orchestrator/Dockerfile]
-
-[bumpversion:file:airbyte-cron/Dockerfile]
-
-[bumpversion:file:airbyte-metrics/reporter/Dockerfile]
-
-[bumpversion:file:airbyte-proxy/Dockerfile]
-
-[bumpversion:file:airbyte-server/Dockerfile]
-
-[bumpversion:file:airbyte-webapp/package-lock.json]
-
-[bumpversion:file:airbyte-webapp/package.json]
-
-[bumpversion:file:airbyte-workers/Dockerfile]
-
-[bumpversion:file:charts/airbyte-bootloader/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-cron/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-server/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-temporal/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-webapp/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-worker/Chart.yaml]
-
-[bumpversion:file:charts/airbyte/Chart.yaml]
-
-[bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml]
-
-[bumpversion:file:charts/airbyte/README.md]
+[bumpversion:file:gradle.properties]
 
 [bumpversion:file:docs/operator-guides/upgrading-airbyte.md]
 
-[bumpversion:file:kube/overlays/stable-with-resource-limits/.env]
-
-[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml]
-
-[bumpversion:file:kube/overlays/stable/.env]
-
-[bumpversion:file:kube/overlays/stable/kustomization.yaml]
-
 [bumpversion:file:octavia-cli/Dockerfile]
 
 [bumpversion:file:octavia-cli/README.md]
 
 [bumpversion:file:octavia-cli/install.sh]
 
 [bumpversion:file:octavia-cli/setup.py]
-
-[bumpversion:file:airbyte-connector-builder-server/Dockerfile]
-
-[bumpversion:file:airbyte-connector-builder-server/setup.py]
+search = version="{current_version}"
+replace = version="{new_version}"

.editorconfig

@@ -137,13 +137,3 @@ ij_yaml_sequence_on_new_line = false
 ij_yaml_space_before_colon = false
 ij_yaml_spaces_within_braces = true
 ij_yaml_spaces_within_brackets = true
-
-# don't interfere with prettier default formatting for webapp
-# see https://prettier.io/docs/en/api.html#prettierresolveconfigfilepath--options
-# for options that we're disabling
-[/airbyte-webapp/**]
-end_of_line = unset
-indent_style = unset
-indent_size = unset
-tab_width = unset
-max_line_length = unset

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.29
+VERSION=0.42.0
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -96,7 +96,7 @@ MAX_SPEC_WORKERS=5
 MAX_CHECK_WORKERS=5
 MAX_DISCOVER_WORKERS=5
 MAX_NOTIFY_WORKERS=5
-SHOULD_RUN_NOTIFY_WORKFLOWS=false
+SHOULD_RUN_NOTIFY_WORKFLOWS=true
 # Temporal Activity configuration
 ACTIVITY_MAX_ATTEMPT=
 ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=
@@ -107,11 +107,17 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=
 AUTO_DISABLE_FAILING_CONNECTIONS=false
 FORCE_MIGRATE_SECRET_STORE=false
 
+FEATURE_FLAG_CLIENT=config
+
 ### MONITORING FLAGS ###
 # Accepted values are datadog and otel (open telemetry)
 METRIC_CLIENT=
 # Useful only when metric client is set to be otel. Must start with http:// or https://.
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
 USE_STREAM_CAPABLE_STATE=true
-AUTO_DETECT_SCHEMA=false
+AUTO_DETECT_SCHEMA=true
+
+# To test local catalog changes, set the below variable to the path of your local catalog.
+# https://docs.airbyte.com/connector-development/testing-connectors/testing-a-local-catalog-in-development
+LOCAL_CONNECTOR_CATALOG_PATH=

.env.dev

@@ -28,7 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
 CRON_MICRONAUT_ENVIRONMENTS=control-plane
-AUTO_DETECT_SCHEMA=false
+AUTO_DETECT_SCHEMA=true
 
 # Sentry
 SENTRY_DSN=""

.github/CODEOWNERS

@@ -1,59 +1,50 @@
-# CDK and SAT
+# CDK and Connector Acceptance Tests
 /airbyte-cdk/ @airbytehq/connector-extensibility
-/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/connector-extensibility
 /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
-
-# Oauth
-/airbyte-oauth/ @airbytehq/connector-operations
-/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @airbytehq/connector-operations
-/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @airbytehq/connector-operations
+/airbyte-integrations/bases/connector-acceptance-tests/ @airbytehq/connector-operations
 
 # Protocol related items
-/airbyte-protocol/ @airbytehq/protocol-reviewers
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
-# Airbyte Maintainers
-airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/ @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/ @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @airbytehq/airbyte-maintainers
-*build.gradle @airbytehq/airbyte-maintainers
-airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors.
-
 # Normalization
-/airbyte-integrations/bases/base-normalization/ @airbytehq/normalization
+/airbyte-integrations/bases/base-normalization/ @airbytehq/destinations
 
-# JDBC-based connectors
+# Java-based connectors
 /airbyte-integrations/bases/base-java/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-jdbc/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-alloydb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-bigquery/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-clickhouse/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-db2/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-mssql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-mysql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-oracle/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-postgres/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-redshift/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-tidb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-jdbc/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-azure-blob-storage/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-clickhouse/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-databricks/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-gcs/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mariadb-columnstore/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mysql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mssql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-oracle/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-postgres/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-redshift/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-rockset/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
+
+# Java-based source connectors
+/airbyte-integrations/bases/debezium-v1-4-2/ @airbytehq/dbsources
+/airbyte-integrations/bases/debezium-v1-9-6/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-jdbc/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-alloydb/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-bigquery/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-clickhouse/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-db2/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-mssql/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-mysql/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-oracle/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-postgres/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-redshift/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-snowflake/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources
+
+# Java-based destination connectors
+/airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
+/airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-jdbc/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-bigquery/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-bigquery-denormalized/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-azure-blob-storage/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-clickhouse/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-databricks/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-gcs/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mariadb-columnstore/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mysql/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mssql/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-oracle/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-postgres/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-rockset/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations

.github/ISSUE_TEMPLATE/low-code-component.yaml

@@ -0,0 +1,39 @@
+name: Low Code Component
+description: Use this when requesting low-code CDK components
+title: "Low-code CDK: "
+labels:
+  [
+    "team/extensibility",
+    "area/connector-builder",
+    "area/low-code/components",
+    "area/low-code",
+  ]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Component Description
+      description: Please describe the component you would like to see added to the Low-code CDK and why it's valuable
+    validations:
+      required: true
+  - type: textarea
+    id: proposed-schema
+    attributes:
+      label: Proposed YAML schema
+      description: If you can wave a magic wand, what would the YAML schema of the component you are suggesting look like?
+      value: |
+        ```yaml
+        ...
+        ```
+    validations:
+      required: true
+  - type: input
+    id: url
+    attributes:
+      label: Connector API Docs URL
+      description: To help us understand your request, please share a link to the API docs for the API you're trying to build, ideally pointing to the specific section of the docs relevant to your use case
+  - type: input
+    id: pr-url
+    attributes:
+      label: Pull Request URL
+      description: If there is a pull request which currently implements a custom component in lieu of having this component in the CDK, please link it

.github/actions/cache-build-artifacts/action.yml

@@ -4,10 +4,6 @@ inputs:
   cache-key:
     description: "Key to use for caching"
     required: true
-  cache-python:
-    description: "Whether to cache Python dependencies. Only relevant if building connector modules. true or false"
-    default: "true"
-    required: true
 
 runs:
   using: "composite"
@@ -31,6 +27,15 @@ runs:
         restore-keys: |
           ${{ inputs.cache-key }}-npm-${{ runner.os }}-
 
+    - name: pnpm Caching
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.local/share/pnpm/store
+        key: ${{ inputs.cache-key }}-pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+        restore-keys: |
+          ${{ inputs.cache-key }}-pnpm-${{ runner.os }}-
+
     # this intentionally does not use restore-keys so we don't mess with gradle caching
     - name: Gradle and Python Caching
       uses: actions/cache@v3

.github/actions/start-aws-runner/action.yml

@@ -18,7 +18,7 @@ inputs:
     default: "subnet-0469a9e68a379c1d3"
     required: true
   security-group-id:
-    default: "sg-0793f3c9413f21970"
+    default: "sg-0793f3c9413f21970,sg-023656a050e7d5634"
     required: true
   label:
     required: false