From 7b0f11bb13f397ed6cf12a6882869d4efb1a5b46 Mon Sep 17 00:00:00 2001 From: Debanjan Date: Mon, 27 Feb 2023 11:55:42 +0530 Subject: [PATCH 1/5] fix: hubspot dict issue --- .../source-hubspot/source_hubspot/source.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index db286e7fd7e5..8d8706b4d1fc 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -15,7 +15,12 @@ from airbyte_cdk.utils.event_timing import create_timer from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests import HTTPError +<<<<<<< HEAD from source_hubspot.constants import API_KEY_CREDENTIALS +======= +from requests.adapters import HTTPAdapter +from urllib3 import Retry +>>>>>>> 537d1816653 (fix: hubspot dict issue) from source_hubspot.streams import ( API, Campaigns, @@ -67,11 +72,15 @@ def get_granted_scopes(self, authenticator): try: access_token = authenticator.get_access_token() url = f"https://api.hubapi.com/oauth/v1/access-tokens/{access_token}" - response = requests.get(url=url) - response.raise_for_status() - response_json = response.json() - granted_scopes = response_json["scopes"] - return granted_scopes + with requests.Session() as s: + s.mount("https://api.hubapi.com/oauth/v1/access-tokens/", HTTPAdapter(max_retries=Retry( + total=10, backoff_factor=2, allowed_methods=None, status_forcelist=[429, 500, 502, 503, 504]))) + response = s.get(url=url) + response = requests.get(url=url) + response.raise_for_status() + response_json = response.json() + granted_scopes = response_json["scopes"] + return granted_scopes except Exception as e: return False, repr(e) From e33deea9b20986e9d1285a34869a65141e4e04a4 Mon Sep 17 00:00:00 2001 From: Debanjan Date: Mon, 27 Feb 2023 13:11:48 +0530 Subject: [PATCH 2/5] fix: imports --- .../connectors/source-hubspot/source_hubspot/source.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 8d8706b4d1fc..5ec09c5a7fb2 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -15,12 +15,9 @@ from airbyte_cdk.utils.event_timing import create_timer from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests import HTTPError -<<<<<<< HEAD from source_hubspot.constants import API_KEY_CREDENTIALS -======= from requests.adapters import HTTPAdapter from urllib3 import Retry ->>>>>>> 537d1816653 (fix: hubspot dict issue) from source_hubspot.streams import ( API, Campaigns, From ae95c5d14a4cc2fb45a1747b89c9b60618701207 Mon Sep 17 00:00:00 2001 From: Debanjan Date: Mon, 27 Feb 2023 13:13:23 +0530 Subject: [PATCH 3/5] miscs --- .../connectors/source-hubspot/source_hubspot/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 5ec09c5a7fb2..56594ae2e58b 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -73,7 +73,6 @@ def get_granted_scopes(self, authenticator): s.mount("https://api.hubapi.com/oauth/v1/access-tokens/", HTTPAdapter(max_retries=Retry( total=10, backoff_factor=2, allowed_methods=None, status_forcelist=[429, 500, 502, 503, 504]))) response = s.get(url=url) - response = requests.get(url=url) response.raise_for_status() response_json = response.json() granted_scopes = response_json["scopes"] From 0ff876614061d8132e15f049fb415d68b9e82163 Mon Sep 17 00:00:00 2001 From: Debanjan Date: Mon, 27 Feb 2023 13:16:16 +0530 Subject: [PATCH 4/5] adds workflow file --- .github/workflows/build-source-image.yaml | 352 ++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 .github/workflows/build-source-image.yaml diff --git a/.github/workflows/build-source-image.yaml b/.github/workflows/build-source-image.yaml new file mode 100644 index 000000000000..6384ce630ae6 --- /dev/null +++ b/.github/workflows/build-source-image.yaml @@ -0,0 +1,352 @@ +name: Build Source Container +on: + workflow_dispatch: + inputs: + connector: + description: 'Connector to build' + required: true + type: choice + options: + - source-activecampaign + - source-adjust + - source-airtable + - source-alloydb + - source-alloydb-strict-encrypt + - source-amazon-ads + - source-amazon-seller-partner + - source-amazon-sqs + - source-amplitude + - source-apify-dataset + - source-appfollow + - source-appsflyer + - source-appstore-singer + - source-asana + - source-ashby + - source-auth0 + - source-aws-cloudtrail + - source-azure-table + - source-bamboo-hr + - source-bigcommerce + - source-bigquery + - source-bing-ads + - source-braintree + - source-cart + - source-chargebee + - source-chargify + - source-chartmogul + - source-clickhouse + - source-clickhouse-strict-encrypt + - source-clockify + - source-close-com + - source-cockroachdb + - source-cockroachdb-strict-encrypt + - source-coin-api + - source-coinmarketcap + - source-commercetools + - source-confluence + - source-convertkit + - source-courier + - source-db2 + - source-db2-strict-encrypt + - source-delighted + - source-dixa + - source-dockerhub + - source-drift + - source-dv-360 + - source-e2e-test + - source-e2e-test-cloud + - source-elasticsearch + - source-exchange-rates + - source-facebook-marketing + - source-facebook-pages + - source-faker + - source-fauna + - source-file + - source-file-secure + - source-firebolt + - source-flexport + - source-freshcaller + - source-freshdesk + - source-freshsales + - source-freshservice + - source-github + - source-gitlab + - source-glassfrog + - source-gocardless + - source-google-ads + - source-google-analytics-data-api + - source-google-analytics-v4 + - source-google-directory + - source-google-search-console + - source-google-sheets + - source-google-webfonts + - source-google-workspace-admin-reports + - source-greenhouse + - source-gutendex + - source-harvest + - source-hellobaton + - source-hubplanner + - source-hubspot + - source-insightly + - source-instagram + - source-intercom + - source-iterable + - source-jdbc + - source-jira + - source-kafka + - source-klaviyo + - source-kustomer-singer + - source-kyriba + - source-lemlist + - source-lever-hiring + - source-linkedin-ads + - source-linkedin-pages + - source-linnworks + - source-lokalise + - source-looker + - source-mailchimp + - source-mailerlite + - source-mailgun + - source-mailjet-mail + - source-mailjet-sms + - source-marketo + - source-metabase + - source-microsoft-teams + - source-mixpanel + - source-monday + - source-mongodb + - source-mongodb-strict-encrypt + - source-mongodb-v2 + - source-mssql + - source-mssql-strict-encrypt + - source-my-hours + - source-mysql + - source-mysql-strict-encrypt + - source-nasa + - source-netsuite + - source-news-api + - source-notion + - source-okta + - source-omnisend + - source-onesignal + - source-openweather + - source-oracle + - source-oracle-strict-encrypt + - source-orb + - source-orbit + - source-oura + - source-outreach + - source-pardot + - source-paypal-transaction + - source-paystack + - source-persistiq + - source-pinterest + - source-pipedrive + - source-pivotal-tracker + - source-plaid + - source-pokeapi + - source-postgres + - source-postgres-strict-encrypt + - source-posthog + - source-prestashop + - source-primetric + - source-public-apis + - source-python-http-tutorial + - source-qualaroo + - source-quickbooks-singer + - source-rd-station-marketing + - source-recharge + - source-recurly + - source-redshift + - source-relational-db + - source-retently + - source-rki-covid + - source-s3 + - source-salesforce + - source-salesloft + - source-scaffold-java-jdbc + - source-scaffold-- source-http + - source-scaffold-- source-python + - source-search-metrics + - source-sendgrid + - source-sentry + - source-sftp + - source-sftp-bulk + - source-shopify + - source-shortio + - source-slack + - source-smartsheets + - source-snapchat-marketing + - source-snowflake + - source-sonar-cloud + - source-square + - source-stock-ticker-api-tutorial + - source-strava + - source-stripe + - source-surveymonkey + - source-talkdesk-explore + - source-tempo + - source-tidb + - source-tiktok-marketing + - source-timely + - source-tplcentral + - source-trello + - source-tvmaze-schedule + - source-twilio + - source-typeform + - source-us-census + - source-waiteraid + - source-webflow + - source-whisky-hunter + - source-woocommerce + - source-workable + - source-wrike + - source-xkcd + - source-yahoo-finance-price + - source-yandex-metrica + - source-youtube-analytics + - source-zendesk-chat + - source-zendesk-sell + - source-zendesk-sunshine + - source-zendesk-support + - source-zendesk-talk + - source-zenefits + - source-zenloop + - source-zoho-crm + - source-zoom + - source-zuora + + customtag: + description: 'Custom tag (If entered, the image will be build using this tag)' + + change_type: + description: 'Change type (only applicable when image build from main, ignored when changes are included in latest tag)' + required: true + type: choice + default: "minor" + options: + - minor + - patch + + sort_by: + description: 'Latest tag by' + required: true + type: choice + default: "version" + options: + - date + - version +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check out repository code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get previous tag + if: ${{ github.event.inputs.customtag == '' }} + id: current_tag + uses: "debanjan97/github-action-get-previous-tag@v1.0.0" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + sort: ${{ github.event.inputs.sort_by }} + stable: true # ignores alpha builds + + - name: Check if version bump is required + if: ${{ github.ref_name == 'main' }} + id: bump + run: | + if [ "${{ github.event.inputs.customtag }}" != "" ]; then + # custom tag exists, no version bump required + echo "required=false" >> $GITHUB_OUTPUT + exit 0 + fi; + + if [ "${{ github.event.inputs.change_type }}" = "n/a" ]; then + echo "change type is marked n/a, assuming no version bump is required" + echo "required=false" >> $GITHUB_OUTPUT + exit 0 + fi; + + last_commit=$(git rev-list -n 1 ${{ steps.current_tag.outputs.tag }}) + current_commit=${{ github.sha }} + if [ $last_commit = $current_commit ]; then + echo "no new commits from the last tag, no version bump is required" + required=false + else + required=true + fi; + echo "required=$required" >> $GITHUB_OUTPUT + + - name: Calculate Next Versions + if: ${{ steps.bump.outputs.required == 'true' }} + id: calculatenextversion + uses: "WyriHaximus/github-action-next-semvers@v1" + with: + version: ${{ steps.current_tag.outputs.tag }} + + - name: Generate New Version according to change_type + if: ${{ steps.bump.outputs.required == 'true' }} + id: newversion + run: | + if [ "${{ github.event.inputs.change_type }}" = "minor" ]; then + newversion=${{ steps.calculatenextversion.outputs.v_minor }} + else + newversion=${{ steps.calculatenextversion.outputs.v_patch }} + fi; + echo "version=$newversion" >> $GITHUB_OUTPUT + + - name: Get Build Tag + id: buildtag + run: | + # if custom tag is present, return it + if [ "${{ github.event.inputs.customtag }}" != "" ]; then + tag="${{ github.event.inputs.customtag }}" + echo "tag=$tag" >> $GITHUB_OUTPUT + exit 0 + fi; + + source_branch_name="${GITHUB_REF##*/}" + if [ $source_branch_name = "main" ]; then + if [ "${{ steps.bump.outputs.required }}" = "false" ]; then + # return current tag, if no new commits + echo "no version bump required, proceeding with current version" + tag=${{ steps.current_tag.outputs.tag }} + else + tag=${{ steps.newversion.outputs.version }} + fi; + else + # if branch is feature branch, append alpha. to the latest tag + git_hash=$(git rev-parse --short "$GITHUB_SHA") + tag=${{ steps.current_tag.outputs.tag }}"-alpha."$git_hash + fi; + echo "tag=$tag" >> $GITHUB_OUTPUT + + - uses: mukunku/tag-exists-action@v1.2.0 + name: Check if the generated tag exists + id: checkTag + with: + tag: ${{ steps.buildtag.outputs.tag }} + + - name: Publish version as a tag + if: ${{ steps.checkTag.outputs.exists != 'true' }} + run: | + git tag ${{ steps.buildtag.outputs.tag }} + git push --tag + - name: Build and Push + uses: docker/build-push-action@v3 + with: + context: ./airbyte-integrations/connectors/${{ github.event.inputs.connector }} + file: ./airbyte-integrations/connectors/${{ github.event.inputs.connector }}/Dockerfile + push: true + platforms: linux/amd64 + tags: rudderstack/${{ github.event.inputs.connector }}:${{ steps.buildtag.outputs.tag }} \ No newline at end of file From 5f169c79d09b991be529530c861f8f6b22cae936 Mon Sep 17 00:00:00 2001 From: Debanjan Date: Fri, 3 Mar 2023 12:10:09 +0530 Subject: [PATCH 5/5] chore: reuse `retry_connection_handler` --- .../source-hubspot/source_hubspot/source.py | 35 ++++++++++--------- .../source-hubspot/source_hubspot/streams.py | 5 +++ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 56594ae2e58b..00acee17fc86 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -3,6 +3,7 @@ # import logging +from itertools import chain from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import requests @@ -15,9 +16,6 @@ from airbyte_cdk.utils.event_timing import create_timer from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests import HTTPError -from source_hubspot.constants import API_KEY_CREDENTIALS -from requests.adapters import HTTPAdapter -from urllib3 import Retry from source_hubspot.streams import ( API, Campaigns, @@ -41,11 +39,12 @@ Owners, Products, PropertyHistory, - Quotes, SubscriptionChanges, TicketPipelines, Tickets, Workflows, + retry_connection_handler, + retry_after_handler, ) @@ -66,17 +65,19 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return alive, error_msg def get_granted_scopes(self, authenticator): + @retry_connection_handler(max_tries=5, factor=1) + @retry_after_handler(max_tries=3) + def _request_scopes(url): + response = requests.get(url=url) + response.raise_for_status() + return response.json() + try: access_token = authenticator.get_access_token() url = f"https://api.hubapi.com/oauth/v1/access-tokens/{access_token}" - with requests.Session() as s: - s.mount("https://api.hubapi.com/oauth/v1/access-tokens/", HTTPAdapter(max_retries=Retry( - total=10, backoff_factor=2, allowed_methods=None, status_forcelist=[429, 500, 502, 503, 504]))) - response = s.get(url=url) - response.raise_for_status() - response_json = response.json() - granted_scopes = response_json["scopes"] - return granted_scopes + response_json = _request_scopes(url) + granted_scopes = response_json["scopes"] + return granted_scopes except Exception as e: return False, repr(e) @@ -122,10 +123,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Workflows(**common_params), ] - credentials_title = credentials.get("credentials_title") - if credentials_title == API_KEY_CREDENTIALS: - streams.append(Quotes(**common_params)) - api = API(credentials=credentials) if api.is_oauth2(): authenticator = API(credentials=credentials).get_authenticator() @@ -135,6 +132,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: available_streams = [stream for stream in streams if stream.scope_is_granted(granted_scopes)] unavailable_streams = [stream for stream in streams if not stream.scope_is_granted(granted_scopes)] self.logger.info(f"The following streams are unavailable: {[s.name for s in unavailable_streams]}") + partially_available_streams = [stream for stream in streams if not stream.properties_scope_is_granted()] + required_scoped = set(chain(*[x.properties_scopes for x in partially_available_streams])) + self.logger.info( + f"The following streams are partially available: {[s.name for s in partially_available_streams]}, " + f"add the following scopes to download all available data: {required_scoped}" + ) else: self.logger.info("No scopes to grant when authenticating with API key.") available_streams = streams diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 8e2c4c6efe28..f6a1d9327c63 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -54,7 +54,10 @@ CUSTOM_FIELD_VALUE_TO_TYPE = {v: k for k, v in CUSTOM_FIELD_TYPE_TO_VALUE.items()} +# strings, when are substrings of error messages should be retried TOKEN_EXPIRED_ERROR = "oauth-token is expired" +TOKEN_REFRESH_RETRIES_EXCEEDED_ERROR = "Max retries exceeded with url: /oauth/v1/token" + def retry_connection_handler(**kwargs): """Retry helper, log each attempt""" @@ -67,6 +70,8 @@ def log_retry_attempt(details): def giveup_handler(exc): if isinstance(exc, HubspotInvalidAuth) and TOKEN_EXPIRED_ERROR in exc.response: return False + if TOKEN_REFRESH_RETRIES_EXCEEDED_ERROR.lower() in exc.response.lower(): + return False if isinstance(exc, (HubspotInvalidAuth, HubspotAccessDenied)): return True return exc.response is not None and HTTPStatus.BAD_REQUEST <= exc.response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR