Support dbt 1.0 02 (#31)

* Restructure how to generate models

* Update

* Update

* Test on Python 3.8 and 3.9

* Don't support 3.9 yet

.github/workflows/checks.yml

@@ -6,6 +6,11 @@ on:
 jobs:
 
   python:
+    strategy:
+      matrix:
+        include:
+          - python_version: 3.8
+
     runs-on: ubuntu-20.04
     env:
       ENV_FILE: ".env/.env.test"
@@ -14,14 +19,14 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python_version }}
       - name: install dependencies
         run: |
           bash ci/setup.sh
       - name: lint
         run: |
           bash ci/lint_python.sh
-      - name:
+      - name: test
         run: |
           export ENV_FILE="${{ env.ENV_FILE }}"
           bash ci/run_python_tests.sh

Makefile

@@ -1,5 +1,6 @@
-DOCKER_IMAGE_BASE = gcr.io/ubie-yu-sandbox/dbt-artifacts-loader
-TAG = "v1.1.0-rc3"
+PROJECT_ID = YOUR-GCP-PROJECT
+TAG = v1.2.0-dev2
+DOCKER_IMAGE_BASE = gcr.io/$(PROJECT_ID)/dbt-artifacts-loader
 
 
 .PHONEY: setup

dbt_artifacts_loader/api/rest_api_v2.py

@@ -28,12 +28,16 @@
 
 from google.cloud import bigquery
 
-from dbt_artifacts_loader.dbt.model_factory import get_model_class
+from dbt_artifacts_loader.dbt.version_map import ArtifactsTypes
 from dbt_artifacts_loader.utils import download_gcs_object_as_text
+from dbt_artifacts_loader.dbt.utils import (
+    get_dbt_schema_version,
+    get_default_load_job_config,
+    load_table_from_json,
+    get_artifact_type_by_id,
+    get_destination_table,
+    get_model_class)
 from dbt_artifacts_loader.api import config
-from dbt_artifacts_loader.dbt.utils import get_dbt_schema_version, get_default_load_job_config, load_table_from_json, \
-    get_artifact_type_by_id, get_destination_table
-from dbt_artifacts_loader.dbt.utils import ArtifactsTypes
 
 app = FastAPI()
 

dbt_artifacts_loader/dbt/base_bigquery_model.py

@@ -18,7 +18,8 @@
 import json
 from enum import Enum
 from typing import Dict, Any, Optional, Union, List
-from datetime import datetime, date
+import datetime
+from datetime import date, datetime
 import inspect
 
 # pylint: disable=E0611
@@ -28,7 +29,12 @@
 
 from google.cloud import bigquery
 
-from dbt_artifacts_loader.dbt.utils import datetime_handler
+
+def datetime_handler(x):
+    """The handler is used to deal with date and datetime"""
+    if isinstance(x, (datetime.datetime, datetime.date, datetime, date)):
+        return x.isoformat()
+    raise TypeError(f'Type {type(x)} not serializable')
 
 
 class TypingUtils:

dbt_artifacts_loader/dbt/utils.py

@@ -15,97 +15,12 @@
 #  limitations under the License.
 #
 #
-from enum import Enum
-import datetime
-from datetime import date, datetime
-from typing import Optional, List
-from dataclasses import dataclass
+from typing import Optional, List, Type
 
 from google.cloud import bigquery
 
-
-class DestinationTables(Enum):
-    # V1
-    CATALOG_V1 = "catalog_v1"
-    MANIFEST_V1 = "manifest_v1"
-    RUN_RESULTS_V1 = "run_results_v1"
-    SOURCES_V1 = "sources_v1"
-    # V2
-    MANIFEST_V2 = "manifest_v2"
-    RUN_RESULTS_V2 = "run_results_v2"
-    SOURCES_V2 = "sources_v2"
-    # V3
-    MANIFEST_V3 = "manifest_v3"
-    RUN_RESULTS_V3 = "run_results_v3"
-    SOURCES_V3 = "sources_v3"
-    # V4
-    MANIFEST_V4 = "manifest_v4"
-    RUN_RESULTS_V4 = "run_results_v4"
-
-
-class ArtifactsTypes(Enum):
-    # V1
-    CATALOG_V1 = "CatalogV1"
-    MANIFEST_V1 = "ManifestV1"
-    RUN_RESULTS_V1 = "RunResultsV1"
-    SOURCES_V1 = "SourcesV1"
-    # V2
-    MANIFEST_V2 = "ManifestV2"
-    RUN_RESULTS_V2 = "RunResultsV2"
-    SOURCES_V2 = "SourcesV2"
-    # V3
-    MANIFEST_V3 = "ManifestV3"
-    RUN_RESULTS_V3 = "RunResultsV3"
-    SOURCES_V3 = "SourcesV3"
-    # V4
-    MANIFEST_V4 = "ManifestV4"
-    RUN_RESULTS_V4 = "RunResultsV4"
-
-
-@dataclass
-class ArtifactInfo:
-    dbt_schema_version: str
-    artifact_type: ArtifactsTypes
-    destination_table: DestinationTables
-
-
-ARTIFACT_INFO = {
-    # V1
-    "CATALOG_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/catalog/v1.json",
-                               ArtifactsTypes.CATALOG_V1, DestinationTables.CATALOG_V1),
-    "MANIFEST_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v1.json",
-                                ArtifactsTypes.MANIFEST_V1, DestinationTables.MANIFEST_V1),
-    "RUN_RESULTS_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v1.json",
-                                   ArtifactsTypes.RUN_RESULTS_V1, DestinationTables.RUN_RESULTS_V1),
-    "SOURCES_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v1.json",
-                               ArtifactsTypes.SOURCES_V1, DestinationTables.SOURCES_V1),
-    # V2
-    "MANIFEST_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v2.json",
-                                ArtifactsTypes.MANIFEST_V2, DestinationTables.MANIFEST_V2),
-    "RUN_RESULTS_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v2.json",
-                                   ArtifactsTypes.RUN_RESULTS_V2, DestinationTables.RUN_RESULTS_V2),
-    "SOURCES_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v2.json",
-                               ArtifactsTypes.SOURCES_V2, DestinationTables.SOURCES_V2),
-    # V3
-    "MANIFEST_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v3.json",
-                                ArtifactsTypes.MANIFEST_V3, DestinationTables.MANIFEST_V3),
-    "RUN_RESULTS_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v3.json",
-                                   ArtifactsTypes.RUN_RESULTS_V3, DestinationTables.RUN_RESULTS_V3),
-    "SOURCES_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v3.json",
-                               ArtifactsTypes.SOURCES_V3, DestinationTables.SOURCES_V3),
-    # V4
-    "MANIFEST_V4": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v4.json",
-                                ArtifactsTypes.MANIFEST_V4, DestinationTables.MANIFEST_V4),
-    "RUN_RESULTS_V4": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v4.json",
-                                   ArtifactsTypes.RUN_RESULTS_V4, DestinationTables.RUN_RESULTS_V4),
-}
-
-
-def datetime_handler(x):
-    """The handler is used to deal with date and datetime"""
-    if isinstance(x, (datetime.datetime, datetime.date, datetime, date)):
-        return x.isoformat()
-    raise TypeError(f'Type {type(x)} not serializable')
+from dbt_artifacts_loader.dbt.base_bigquery_model import BaseBigQueryModel
+from dbt_artifacts_loader.dbt.version_map import ArtifactsTypes, ARTIFACT_INFO, DestinationTables
 
 
 def get_dbt_schema_version(artifact_json: dict) -> str:
@@ -178,3 +93,19 @@ def load_table_from_json(
     # pylint: disable=W0703
     except Exception as e:
         raise RuntimeError(job.errors) from e
+
+
+def get_model_class(artifact_type: ArtifactsTypes) -> Type[BaseBigQueryModel]:
+    """Get the model class
+
+    Args:
+        artifact_type (ArtifactsTypes): artifact type
+
+    Returns:
+        the model class
+    """
+    # v1
+    for _, artifact_info in ARTIFACT_INFO.items():
+        if artifact_type == artifact_info.artifact_type:
+            return artifact_info.model_class
+    raise ValueError(f"No such an artifact {artifact_type}")

dbt_artifacts_loader/dbt/version_map.py

@@ -0,0 +1,114 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from typing import Type
+from enum import Enum
+from dataclasses import dataclass
+
+from dbt_artifacts_loader.dbt.base_bigquery_model import BaseBigQueryModel
+
+from dbt_artifacts_loader.dbt.v1.catalog import CatalogV1
+from dbt_artifacts_loader.dbt.v1.manifest import ManifestV1
+from dbt_artifacts_loader.dbt.v1.run_results import RunResultsV1
+from dbt_artifacts_loader.dbt.v1.sources import SourcesV1
+from dbt_artifacts_loader.dbt.v2.manifest import ManifestV2
+from dbt_artifacts_loader.dbt.v2.run_results import RunResultsV2
+from dbt_artifacts_loader.dbt.v2.sources import SourcesV2
+from dbt_artifacts_loader.dbt.v3.manifest import ManifestV3
+from dbt_artifacts_loader.dbt.v3.run_results import RunResultsV3
+from dbt_artifacts_loader.dbt.v3.sources import SourcesV3
+from dbt_artifacts_loader.dbt.v4.manifest import ManifestV4
+from dbt_artifacts_loader.dbt.v4.run_results import RunResultsV4
+
+
+class DestinationTables(Enum):
+    """Destination tables"""
+    # V1
+    CATALOG_V1 = "catalog_v1"
+    MANIFEST_V1 = "manifest_v1"
+    RUN_RESULTS_V1 = "run_results_v1"
+    SOURCES_V1 = "sources_v1"
+    # V2
+    MANIFEST_V2 = "manifest_v2"
+    RUN_RESULTS_V2 = "run_results_v2"
+    SOURCES_V2 = "sources_v2"
+    # V3
+    MANIFEST_V3 = "manifest_v3"
+    RUN_RESULTS_V3 = "run_results_v3"
+    SOURCES_V3 = "sources_v3"
+    # V4
+    MANIFEST_V4 = "manifest_v4"
+    RUN_RESULTS_V4 = "run_results_v4"
+
+
+class ArtifactsTypes(Enum):
+    """Dbt artifacts types"""
+    # V1
+    CATALOG_V1 = "CatalogV1"
+    MANIFEST_V1 = "ManifestV1"
+    RUN_RESULTS_V1 = "RunResultsV1"
+    SOURCES_V1 = "SourcesV1"
+    # V2
+    MANIFEST_V2 = "ManifestV2"
+    RUN_RESULTS_V2 = "RunResultsV2"
+    SOURCES_V2 = "SourcesV2"
+    # V3
+    MANIFEST_V3 = "ManifestV3"
+    RUN_RESULTS_V3 = "RunResultsV3"
+    SOURCES_V3 = "SourcesV3"
+    # V4
+    MANIFEST_V4 = "ManifestV4"
+    RUN_RESULTS_V4 = "RunResultsV4"
+
+
+@dataclass
+class ArtifactInfo:
+    dbt_schema_version: str
+    artifact_type: ArtifactsTypes
+    destination_table: DestinationTables
+    model_class: Type[BaseBigQueryModel]
+
+
+ARTIFACT_INFO = {
+    # V1
+    "CATALOG_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/catalog/v1.json",
+                               ArtifactsTypes.CATALOG_V1, DestinationTables.CATALOG_V1, CatalogV1),
+    "MANIFEST_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v1.json",
+                                ArtifactsTypes.MANIFEST_V1, DestinationTables.MANIFEST_V1, ManifestV1),
+    "RUN_RESULTS_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v1.json",
+                                   ArtifactsTypes.RUN_RESULTS_V1, DestinationTables.RUN_RESULTS_V1, RunResultsV1),
+    "SOURCES_V1": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v1.json",
+                               ArtifactsTypes.SOURCES_V1, DestinationTables.SOURCES_V1, SourcesV1),
+    # V2
+    "MANIFEST_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v2.json",
+                                ArtifactsTypes.MANIFEST_V2, DestinationTables.MANIFEST_V2, ManifestV2),
+    "RUN_RESULTS_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v2.json",
+                                   ArtifactsTypes.RUN_RESULTS_V2, DestinationTables.RUN_RESULTS_V2, RunResultsV2),
+    "SOURCES_V2": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v2.json",
+                               ArtifactsTypes.SOURCES_V2, DestinationTables.SOURCES_V2, SourcesV2),
+    # V3
+    "MANIFEST_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v3.json",
+                                ArtifactsTypes.MANIFEST_V3, DestinationTables.MANIFEST_V3, ManifestV3),
+    "RUN_RESULTS_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v3.json",
+                                   ArtifactsTypes.RUN_RESULTS_V3, DestinationTables.RUN_RESULTS_V3, RunResultsV3),
+    "SOURCES_V3": ArtifactInfo("https://schemas.getdbt.com/dbt/sources/v3.json",
+                               ArtifactsTypes.SOURCES_V3, DestinationTables.SOURCES_V3, SourcesV3),
+    # V4
+    "MANIFEST_V4": ArtifactInfo("https://schemas.getdbt.com/dbt/manifest/v4.json",
+                                ArtifactsTypes.MANIFEST_V4, DestinationTables.MANIFEST_V4, ManifestV4),
+    "RUN_RESULTS_V4": ArtifactInfo("https://schemas.getdbt.com/dbt/run-results/v4.json",
+                                   ArtifactsTypes.RUN_RESULTS_V4, DestinationTables.RUN_RESULTS_V4, RunResultsV4),
+}

terraform/example/main.tf

@@ -6,7 +6,7 @@ module "dbt_artifacts_loader" {
 
   delete_on_destroy = true
 
-  docker_image = "gcr.io/${var.project_id}/dbt-artifacts-loader:v1.1.0-rc2"
+  docker_image = "gcr.io/${var.project_id}/dbt-artifacts-loader:v1.2.0-dev2"
 
   labels = {
     app = "dbt-artifacts-loader"