Subtitle and subjects update (#241)

The-Academic-Observatory · Jul 2, 2024 · 10d178b · 10d178b
1 parent a1846bb
commit 10d178b
Show file tree

Hide file tree

Showing 26 changed files with 491 additions and 97 deletions.
diff --git a/dags/oaebu_workflows/onix_workflow/onix_workflow.py b/dags/oaebu_workflows/onix_workflow/onix_workflow.py
@@ -686,6 +686,9 @@ def create_book_product_table(release: dict, **context) -> None:
             )
 
             # Render the SQL
+            bic_table_id = bq_table_id(bq_subject_project_id, bq_subject_dataset_id, "bic_lookup")
+            thema_table_id = bq_table_id(bq_subject_project_id, bq_subject_dataset_id, "thema_lookup")
+            bisac_table_id = bq_table_id(bq_subject_project_id, bq_subject_dataset_id, "bisac_lookup")
             env = create_data_partner_env(
                 main_template=os.path.join(sql_folder(workflow_module="onix_workflow"), "book_product.sql.jinja2"),
                 data_partners=data_partners,
@@ -698,6 +701,9 @@ def create_book_product_table(release: dict, **context) -> None:
                 workid_table_id=workid_table_id,
                 workfamilyid_table_id=workfamilyid_table_id,
                 ga3_views_field=ga3_views_field,
+                bic_table_id=bic_table_id,
+                thema_table_id=thema_table_id,
+                bisac_table_id=bisac_table_id,
                 **dp_tables,
             )
             logging.info(f"Book Product SQL:\n{sql}")

diff --git a/dags/oaebu_workflows/onix_workflow/schema/book_list.json b/dags/oaebu_workflows/onix_workflow/schema/book_list.json
@@ -57,7 +57,19 @@
     "mode": "NULLABLE",
     "name": "title",
     "type": "STRING",
-    "description": "The Books Title"
+    "description": "The Book's Title"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "subtitle",
+    "type": "STRING",
+    "description": "Subtitle of the Book"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "title_subtitle",
+    "type": "STRING",
+    "description": "The concatenated title and subtitle of the Book"
   },
   {
     "mode": "REPEATED",
@@ -69,27 +81,63 @@
     "fields": [
       {
         "mode": "REPEATED",
-        "name": "bic",
+        "name": "bic_codes",
+        "type": "STRING",
+        "description": "A list of BIC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_top",
+        "type": "STRING",
+        "description": "A list of BIC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_names",
+        "type": "STRING",
+        "description": "A list of BIC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_codes",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_top",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_names",
+        "type": "STRING",
+        "description": "A list of BISAC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "thema_codes",
         "type": "STRING",
-        "description": "BIC subject codes"
+        "description": "A list of Thema subject codes"
       },
       {
         "mode": "REPEATED",
-        "name": "bisac",
+        "name": "thema_top",
         "type": "STRING",
-        "description": "BISAC subject codes"
+        "description": "A list of Thema subject codes for the top level subject"
       },
       {
         "mode": "REPEATED",
-        "name": "thema",
+        "name": "thema_names",
         "type": "STRING",
-        "description": "THEMA subject codes"
+        "description": "A list of Thema subject names"
       }
     ],
     "mode": "NULLABLE",
     "name": "subjects",
     "type": "RECORD",
-    "description": "Subject codes"
+    "description": "Subjects associated with this product"
   },
   {
     "fields": [

diff --git a/dags/oaebu_workflows/onix_workflow/schema/book_metrics.json b/dags/oaebu_workflows/onix_workflow/schema/book_metrics.json
@@ -23,31 +23,79 @@
     "type": "STRING",
     "description": "The title of the book"
   },
+  {
+    "mode": "NULLABLE",
+    "name": "subtitle",
+    "type": "STRING",
+    "description": "Subtitle of the Book"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "title_subtitle",
+    "type": "STRING",
+    "description": "The concatenated title and subtitle of the Book"
+  },
   {
     "fields": [
       {
         "mode": "REPEATED",
-        "name": "bic",
+        "name": "bic_codes",
+        "type": "STRING",
+        "description": "A list of BIC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_top",
+        "type": "STRING",
+        "description": "A list of BIC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_names",
+        "type": "STRING",
+        "description": "A list of BIC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_codes",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_top",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_names",
+        "type": "STRING",
+        "description": "A list of BISAC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "thema_codes",
         "type": "STRING",
-        "description": "BIC subject codes"
+        "description": "A list of Thema subject codes"
       },
       {
         "mode": "REPEATED",
-        "name": "bisac",
+        "name": "thema_top",
         "type": "STRING",
-        "description": "BISAC subject codes"
+        "description": "A list of Thema subject codes for the top level subject"
       },
       {
         "mode": "REPEATED",
-        "name": "thema",
+        "name": "thema_names",
         "type": "STRING",
-        "description": "THEMA subject codes"
+        "description": "A list of Thema subject names"
       }
     ],
     "mode": "NULLABLE",
     "name": "subjects",
     "type": "RECORD",
-    "description": "Subject codes"
+    "description": "Subjects associated with this product"
   },
   {
     "fields": [

diff --git a/dags/oaebu_workflows/onix_workflow/schema/book_metrics_city.json b/dags/oaebu_workflows/onix_workflow/schema/book_metrics_city.json
@@ -23,6 +23,18 @@
     "type": "STRING",
     "description": "The title of the book"
   },
+  {
+    "mode": "NULLABLE",
+    "name": "subtitle",
+    "type": "STRING",
+    "description": "Subtitle of the Book"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "title_subtitle",
+    "type": "STRING",
+    "description": "The concatenated title and subtitle of the Book"
+  },
   {
     "mode": "NULLABLE",
     "name": "published_year",
@@ -63,27 +75,63 @@
     "fields": [
       {
         "mode": "REPEATED",
-        "name": "bic",
+        "name": "bic_codes",
+        "type": "STRING",
+        "description": "A list of BIC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_top",
+        "type": "STRING",
+        "description": "A list of BIC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_names",
+        "type": "STRING",
+        "description": "A list of BIC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_codes",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_top",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_names",
+        "type": "STRING",
+        "description": "A list of BISAC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "thema_codes",
         "type": "STRING",
-        "description": "BIC subject codes"
+        "description": "A list of Thema subject codes"
       },
       {
         "mode": "REPEATED",
-        "name": "bisac",
+        "name": "thema_top",
         "type": "STRING",
-        "description": "BISAC subject codes"
+        "description": "A list of Thema subject codes for the top level subject"
       },
       {
         "mode": "REPEATED",
-        "name": "thema",
+        "name": "thema_names",
         "type": "STRING",
-        "description": "THEMA subject codes"
+        "description": "A list of Thema subject names"
       }
     ],
     "mode": "NULLABLE",
     "name": "subjects",
     "type": "RECORD",
-    "description": "Subject codes"
+    "description": "Subjects associated with this product"
   },
   {
     "fields": [

diff --git a/dags/oaebu_workflows/onix_workflow/schema/book_metrics_country.json b/dags/oaebu_workflows/onix_workflow/schema/book_metrics_country.json
@@ -23,6 +23,18 @@
     "type": "STRING",
     "description": "The title of the book"
   },
+  {
+    "mode": "NULLABLE",
+    "name": "subtitle",
+    "type": "STRING",
+    "description": "Subtitle of the Book"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "title_subtitle",
+    "type": "STRING",
+    "description": "The concatenated title and subtitle of the Book"
+  },
   {
     "mode": "NULLABLE",
     "name": "published_year",
@@ -51,27 +63,63 @@
     "fields": [
       {
         "mode": "REPEATED",
-        "name": "bic",
+        "name": "bic_codes",
+        "type": "STRING",
+        "description": "A list of BIC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_top",
+        "type": "STRING",
+        "description": "A list of BIC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bic_names",
+        "type": "STRING",
+        "description": "A list of BIC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_codes",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_top",
+        "type": "STRING",
+        "description": "A list of BISAC subject codes for the top level subject"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "bisac_names",
+        "type": "STRING",
+        "description": "A list of BISAC subject names"
+      },
+      {
+        "mode": "REPEATED",
+        "name": "thema_codes",
         "type": "STRING",
-        "description": "BIC subject codes"
+        "description": "A list of Thema subject codes"
       },
       {
         "mode": "REPEATED",
-        "name": "bisac",
+        "name": "thema_top",
         "type": "STRING",
-        "description": "BISAC subject codes"
+        "description": "A list of Thema subject codes for the top level subject"
       },
       {
         "mode": "REPEATED",
-        "name": "thema",
+        "name": "thema_names",
         "type": "STRING",
-        "description": "THEMA subject codes"
+        "description": "A list of Thema subject names"
       }
     ],
     "mode": "NULLABLE",
     "name": "subjects",
     "type": "RECORD",
-    "description": "Subject codes"
+    "description": "Subjects associated with this product"
   },
   {
     "fields": [