Partial impletmentation of sql and schemas

dags/oaebu_workflows/schema/shmp_pilot/generic.json

@@ -0,0 +1,26 @@
+[
+  {
+    "mode": "REQUIRED",
+    "name": "ISBN13",
+    "type": "STRING",
+    "description": "13 Digit ISBN of the book."
+  },
+  {
+    "mode": "REQUIRED",
+    "name": "month",
+    "type": "DATE",
+    "description": "Start date for period of analytics info."
+  },
+  {
+    "mode": "REQUIRED",
+    "name": "value",
+    "type": "INTEGER",
+    "description": "Measured value."
+  },
+  {
+    "mode": "REQUIRED",
+    "name": "release_date",
+    "type": "DATE",
+    "description": "The end date of the release month"
+  }
+]

dags/oaebu_workflows/schema/shmp_pilot/muse.json

@@ -0,0 +1,122 @@
+[
+  {
+    "description": "The id of the book.",
+    "mode": "NULLABLE",
+    "name": "ID",
+    "type": "STRING"
+  },
+  {
+    "description": "Publication year.",
+    "mode": "NULLABLE",
+    "name": "YEAR",
+    "type": "INTEGER"
+  },
+  {
+    "description": "Publication month.",
+    "mode": "NULLABLE",
+    "name": "MONTH",
+    "type": "INTEGER"
+  },
+  {
+    "description": "Resource type.",
+    "mode": "NULLABLE",
+    "name": "RESOURCE_TYPE",
+    "type": "STRING"
+  },
+  {
+    "description": "Resource id.",
+    "mode": "NULLABLE",
+    "name": "RESOURCE_ID",
+    "type": "STRING"
+  },
+  {
+    "description": "ISBN of the book on MUSE.",
+    "mode": "NULLABLE",
+    "name": "ISBN",
+    "type": "STRING"
+  },
+  {
+    "description": "Title of the book.",
+    "mode": "NULLABLE",
+    "name": "RESOURCE",
+    "type": "STRING"
+  },
+  {
+    "description": "URL of the book.",
+    "mode": "NULLABLE",
+    "name": "RESOURCE_URL",
+    "type": "STRING"
+  },
+  {
+    "description": "Date of launch.",
+    "mode": "NULLABLE",
+    "name": "RESOURCE_LAUNCH",
+    "type": "DATE"
+  },
+  {
+    "description": "Author of the book.",
+    "mode": "NULLABLE",
+    "name": "AUTHOR",
+    "type": "STRING"
+  },
+  {
+    "description": "Title of the chapter.",
+    "mode": "NULLABLE",
+    "name": "FULLTEXT_TITLE",
+    "type": "STRING"
+  },
+  {
+    "description": "URL of the chapter.",
+    "mode": "NULLABLE",
+    "name": "FULLTEXT_URL",
+    "type": "STRING"
+  },
+  {
+    "description": "Date of fulltext launch.",
+    "mode": "NULLABLE",
+    "name": "FULLTEXT_LAUNCH",
+    "type": "DATE"
+  },
+  {
+    "description": "Issue.",
+    "mode": "NULLABLE",
+    "name": "ISSUE",
+    "type": "STRING"
+  },
+  {
+    "description": "Format.",
+    "mode": "NULLABLE",
+    "name": "FORMAT",
+    "type": "STRING"
+  },
+  {
+    "description": "Access type.",
+    "mode": "NULLABLE",
+    "name": "ACCESS",
+    "type": "STRING"
+  },
+  {
+    "description": "Country Name.",
+    "mode": "NULLABLE",
+    "name": "COUNTRY",
+    "type": "STRING"
+  },
+  {
+    "description": "Institution name.",
+    "mode": "NULLABLE",
+    "name": "INSTITUTION",
+    "type": "STRING"
+  },
+  {
+    "description": "Number of requests.",
+    "mode": "NULLABLE",
+    "name": "REQUESTS",
+    "type": "INTEGER"
+  },
+  {
+    "mode": "REQUIRED",
+    "name": "release_date",
+    "type": "DATE",
+    "description": "Last day of the release month. Table is partitioned on this column."
+  }
+]

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_altmetrics.sql

@@ -0,0 +1 @@
+STRUCT(month.altmetrics.total_downloads) AS altmetrics

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_amazon.sql

@@ -0,0 +1 @@
+STRUCT(month.amazon.total_downloads) AS amazon

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_amazon_ltd.sql

@@ -0,0 +1 @@
+STRUCT(month.amazon_ltd.total_downloads) AS amazon_ltd

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_internet_archive.sql

@@ -0,0 +1 @@
+STRUCT(month.internet_archive.total_downloads) AS internet_archive

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_scholarcommons_ltd.sql

@@ -0,0 +1 @@
+STRUCT(month.scholarcommons_ltd.total_downloads) AS scholarcommons_ltd

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_scholarspace_downloads.sql

@@ -0,0 +1 @@
+STRUCT(month.scholarspace_downloads.total_downloads) AS scholarspace_downloads

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_scholarspace_views.sql

@@ -0,0 +1 @@
+STRUCT(month.scholarspace_views.total_downloads) AS scholarspace_views

dags/oaebu_workflows/sql/shmp_pilot/book_metrics_sci_open.sql

@@ -0,0 +1 @@
+STRUCT(month.sci_open.total_downloads) AS sci_open

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_altmetrics.sql.jinja2

@@ -0,0 +1,11 @@
+altmetrics_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ altmetrics_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_amazon.sql.jinja2

@@ -0,0 +1,11 @@
+amazon_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ amazon_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_amazon_ltd.sql.jinja

@@ -0,0 +1,11 @@
+amazon_ltd_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ amazon_ltd_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_internet_archive.sql.jinja2

@@ -0,0 +1,11 @@
+internet_archive_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ internet_archive_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_scholarcommons_ltd.sql.jinja2

@@ -0,0 +1,11 @@
+scholarcommons_ltd_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ scholarcommons_ltd_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_scholarspace_downloads.sql.jinja2

@@ -0,0 +1,11 @@
+scholarspace_downloads_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ scholarspace_downloads_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_scholarspace_views.sql.jinja2

@@ -0,0 +1,11 @@
+scholarspace_views_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ scholarspace_views_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/book_product_body_sci_open.sql.jinja2

@@ -0,0 +1,11 @@
+altmetrics_metrics as (
+    SELECT
+        ISBN13,
+        release_date,
+        value,
+    FROM
+        `{{ altmetrics_table_id }}`
+    GROUP BY
+        ISBN13,
+        release_date
+),

dags/oaebu_workflows/sql/shmp_pilot/month_null_altmetrics.sql

@@ -0,0 +1 @@
+month.altmetrics IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_amazon.sql

@@ -0,0 +1 @@
+month.amazon IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_amazon_ltd.sql

@@ -0,0 +1 @@
+month.amazon_ltd IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_internet_archive.sql

@@ -0,0 +1 @@
+month.internet_archive IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_scholarspace_commons_ltd.sql

@@ -0,0 +1 @@
+month.scholarspace_commons_ltd IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_scholarspace_downloads.sql

@@ -0,0 +1 @@
+month.scholarspace_downloads IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_scholarspace_views.sql

@@ -0,0 +1 @@
+month.scholarspace_views IS NOT NULL

dags/oaebu_workflows/sql/shmp_pilot/month_null_sci_open.sql

@@ -0,0 +1 @@
+month.sci_open IS NOT NULL