Skip to content

Commit

Permalink
Some workflow fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
keegansmith21 committed May 16, 2024
1 parent ec50a9e commit 0ce4857
Show file tree
Hide file tree
Showing 17 changed files with 137 additions and 185 deletions.
2 changes: 1 addition & 1 deletion dags/oaebu_workflows/onix_workflow/onix_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import logging
import os
import re
from typing import Iterable, List, Optional, Tuple, Union
from typing import Iterable, List, Optional, Tuple, Union, Dict

import jsonlines
import pendulum
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "altmetrics_pilot",
"type": "RECORD",
"description": "Metrics derived from altmetrics"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "amazon_ltd_pilot",
"type": "RECORD",
"description": "Metrics derived from amazon_ltd"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "amazon_pilot",
"type": "RECORD",
"description": "Metrics derived from amazon"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "internet_archive_pilot",
"type": "RECORD",
"description": "Metrics derived from internet_archive"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "scholarcommons_ltd_pilot",
"type": "RECORD",
"description": "Metrics derived from scholarcommons_ltd"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "scholarspace_downloads_pilot",
"type": "RECORD",
"description": "Metrics derived from scholarspace_downloads"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "scholarspace_views_pilot",
"type": "RECORD",
"description": "Metrics derived from scholarspace_views"
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
[
{
"mode": "REQUIRED",
"name": "ISBN13",
"type": "STRING",
"description": "13 Digit ISBN of the book."
},
{
"mode": "REQUIRED",
"name": "value",
"type": "INTEGER",
"description": "Measured value."
},
{
"mode": "REQUIRED",
"name": "release_date",
"type": "DATE",
"description": "The end date of the release month"
}
]
{
"fields": [
{
"mode": "NULLABLE",
"name": "value",
"type": "INTEGER",
"description": "Metric value"
}
],
"mode": "NULLABLE",
"name": "sci_open_pilot",
"type": "RECORD",
"description": "Metrics derived from sci_open"
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
altmetrics_metrics as (
altmetrics_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ altmetrics_table_id }}`
`{{ altmetrics_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
amazon_ltd_metrics as (
amazon_ltd_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ amazon_ltd_table_id }}`
`{{ amazon_ltd_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
amazon_metrics as (
amazon_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ amazon_table_id }}`
`{{ amazon_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
internet_archive_metrics as (
internet_archive_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ internet_archive_table_id }}`
`{{ internet_archive_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
scholarcommons_ltd_metrics as (
scholarcommons_ltd_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ scholarcommons_ltd_table_id }}`
`{{ scholarcommons_ltd_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
scholarspace_downloads_metrics as (
scholarspace_downloads_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ scholarspace_downloads_table_id }}`
`{{ scholarspace_downloads_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
scholarspace_views_metrics as (
scholarspace_views_pilot_metrics as (
SELECT
ISBN13,
release_date,
value,
FROM
`{{ scholarspace_views_table_id }}`
`{{ scholarspace_views_pilot_table_id }}`
GROUP BY
ISBN13,
release_date
),
)
Loading

0 comments on commit 0ce4857

Please sign in to comment.