Toward v1.1.0 rc3 (#24)

* Toward v1.1.0-rc3

* Improve model_factory.py

Makefile

@@ -1,5 +1,5 @@
 DOCKER_IMAGE_BASE = gcr.io/ubie-yu-sandbox/dbt-artifacts-loader
-TAG = "v1.0.0-rc5"
+TAG = "v1.1.0-rc3"
 
 
 .PHONEY: setup

dbt_artifacts_loader/api/rest_api_v2.py

@@ -70,9 +70,9 @@ def is_available(artifact_type: ArtifactsTypes):
         ArtifactsTypes.RUN_RESULTS_V1, ArtifactsTypes.SOURCES_V1,
         # v2
         ArtifactsTypes.RUN_RESULTS_V2, ArtifactsTypes.MANIFEST_V2,
-        ArtifactsTypes.SOURCES_V1,
+        ArtifactsTypes.SOURCES_V2,
         # v3
-        ArtifactsTypes.RUN_RESULTS_V2, ArtifactsTypes.MANIFEST_V2,
+        ArtifactsTypes.RUN_RESULTS_V3, ArtifactsTypes.MANIFEST_V3,
     ]
     if artifact_type in available_artifact_types:
         return True
@@ -126,25 +126,36 @@ def insert_artifact_v2(request_body: RequestBody, settings: config.APISettings =
     if data["contentType"] != "application/json":
         return {"message": "gs://{}/{} is not application/json".format(bucket, name)}
 
+    # Download the file from GCS
     try:
         print("Download gs://{}/{}".format(bucket, name))
         artifact_json_text = download_gcs_object_as_text(project=client_project, bucket=bucket, name=name)
         print("artifact_json_text: {}".format(artifact_json_text))
+    except Exception as e:
+        detail = f"Can not download the GCS object gs://{bucket}/{name}: {str(e)}"
+        print(detail)
+        raise HTTPException(status_code=500, detail=detail) from e
+
+    # Instantiate the JSON object
+    try:
         artifact_json = json.loads(artifact_json_text)
     except Exception as e:
-        detail = "Can not download the GCS object gs://{}/{}: {}".format(bucket, name, str(e))
+        detail = "Can not load the JSON text gs://{}/{}: {}".format(bucket, name, str(e))
+        print(detail)
         raise HTTPException(status_code=500, detail=detail) from e
 
     # Check if the JSON file is one of dbt artifacts types.
     try:
         dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
     except ValueError as e:
         detail = f"{name} is a non-supported JSON file."
+        print(detail)
         raise HTTPException(status_code=500, detail=detail) from e
     print(dbt_schema_version)
     artifact_type = ArtifactsTypes.get_artifact_type_by_id(dbt_schema_version=dbt_schema_version)
     if is_available(artifact_type=artifact_type) is False:
         detail = "gs://{}/{} is not a supported artifact".format(bucket, name)
+        print(detail)
         raise HTTPException(status_code=500, detail=detail)
 
     # Insert a dbt artifact JSON
@@ -173,6 +184,7 @@ def insert_artifact_v2(request_body: RequestBody, settings: config.APISettings =
             # TODO fix the logger
             # logger.error({"message": str(e)})
             detail = "Can not insert artifact to {}: {}".format(full_destination_table_id, str(e))
+            print(detail)
             raise HTTPException(status_code=500, detail=detail) from e
 
     # Construct a response

dbt_artifacts_loader/dbt/model_factory.py

@@ -21,6 +21,9 @@
 from dbt_artifacts_loader.dbt.v1.sources import SourcesV1
 from dbt_artifacts_loader.dbt.v2.manifest import ManifestV2
 from dbt_artifacts_loader.dbt.v2.run_results import RunResultsV2
+from dbt_artifacts_loader.dbt.v2.sources import SourcesV2
+from dbt_artifacts_loader.dbt.v3.manifest import ManifestV3
+from dbt_artifacts_loader.dbt.v3.run_results import RunResultsV3
 
 
 def get_model_class(artifact_type: ArtifactsTypes):
@@ -46,4 +49,11 @@ def get_model_class(artifact_type: ArtifactsTypes):
         return ManifestV2
     elif artifact_type == ArtifactsTypes.RUN_RESULTS_V2:
         return RunResultsV2
+    elif artifact_type == ArtifactsTypes.SOURCES_V2:
+        return SourcesV2
+    # v3
+    elif artifact_type == ArtifactsTypes.MANIFEST_V3:
+        return ManifestV3
+    elif artifact_type == ArtifactsTypes.RUN_RESULTS_V3:
+        return RunResultsV3
     raise ValueError(f"No such an artifact {artifact_type}")

terraform/example/main.tf

@@ -6,7 +6,7 @@ module "dbt_artifacts_loader" {
 
   delete_on_destroy = true
 
-  docker_image = "gcr.io/${var.project_id}/dbt-artifacts-loader:v1.0.0-rc5"
+  docker_image = "gcr.io/${var.project_id}/dbt-artifacts-loader:v1.1.0-rc2"
 
   labels = {
     app = "dbt-artifacts-loader"

tests/dbt/test_utils.py

@@ -27,21 +27,37 @@
 class TestDbtUtils(unittest.TestCase):
 
     def test_get_dbt_schema_version(self):
-        # catalog.json
-        path = os.path.join(get_project_root(), "tests", "resources", "v1", "jaffle_shop", "catalog.json")
-        with open(path, "r", encoding="utf-8") as fp:
-            artifact_json = json.load(fp)
-            dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
-            self.assertEqual(dbt_schema_version, "https://schemas.getdbt.com/dbt/catalog/v1.json")
-        # manifest.json
-        path = os.path.join(get_project_root(), "tests", "resources", "v1", "jaffle_shop", "manifest.json")
-        with open(path, "r", encoding="utf-8") as fp:
-            artifact_json = json.load(fp)
-            dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
-            self.assertEqual(dbt_schema_version, "https://schemas.getdbt.com/dbt/manifest/v1.json")
-        # run_results.json
-        path = os.path.join(get_project_root(), "tests", "resources", "v1", "jaffle_shop", "run_results.json")
-        with open(path, "r", encoding="utf-8") as fp:
-            artifact_json = json.load(fp)
-            dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
-            self.assertEqual(dbt_schema_version, "https://schemas.getdbt.com/dbt/run-results/v1.json")
+        # v1
+        v1_artifacts = {
+            "catalog.json": "https://schemas.getdbt.com/dbt/catalog/v1.json",
+            "manifest.json": "https://schemas.getdbt.com/dbt/manifest/v1.json",
+            "run_results.json": "https://schemas.getdbt.com/dbt/run-results/v1.json",
+        }
+        for file, expected_dbt_schema_version in v1_artifacts.items():
+            path = os.path.join(get_project_root(), "tests", "resources", "v1", "jaffle_shop", file)
+            with open(path, "r", encoding="utf-8") as fp:
+                artifact_json = json.load(fp)
+                dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
+                self.assertEqual(dbt_schema_version, expected_dbt_schema_version)
+        # v2
+        v1_artifacts = {
+            "manifest.json": "https://schemas.getdbt.com/dbt/manifest/v2.json",
+            "run_results.json": "https://schemas.getdbt.com/dbt/run-results/v2.json",
+        }
+        for file, expected_dbt_schema_version in v1_artifacts.items():
+            path = os.path.join(get_project_root(), "tests", "resources", "v2", "jaffle_shop", file)
+            with open(path, "r", encoding="utf-8") as fp:
+                artifact_json = json.load(fp)
+                dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
+                self.assertEqual(dbt_schema_version, expected_dbt_schema_version)
+        # v3
+        v1_artifacts = {
+            "manifest.json": "https://schemas.getdbt.com/dbt/manifest/v3.json",
+            "run_results.json": "https://schemas.getdbt.com/dbt/run-results/v3.json",
+        }
+        for file, expected_dbt_schema_version in v1_artifacts.items():
+            path = os.path.join(get_project_root(), "tests", "resources", "v3", "jaffle_shop", file)
+            with open(path, "r", encoding="utf-8") as fp:
+                artifact_json = json.load(fp)
+                dbt_schema_version = get_dbt_schema_version(artifact_json=artifact_json)
+                self.assertEqual(dbt_schema_version, expected_dbt_schema_version)