Add filter on SubjectCode to avoid nulls in array

The-Academic-Observatory · Oct 21, 2024 · f791522 · f791522
1 parent ea4d118
commit f791522
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/dags/oaebu_workflows/onix_workflow/sql/book_product.sql.jinja2 b/dags/oaebu_workflows/onix_workflow/sql/book_product.sql.jinja2
@@ -92,17 +92,20 @@ onix_ebook_titles_raw as (
                 SELECT
                     subject.SubjectCode
                 FROM UNNEST(onix.Subjects) as subject
-                WHERE subject.SubjectSchemeIdentifier = "BIC_subject_category") as bic_subjects,
+                WHERE subject.SubjectSchemeIdentifier = "BIC_subject_category" 
+                AND subject.SubjectCode IS NOT NULL) as bic_subjects,
             ARRAY(
                 SELECT
                     subject.SubjectCode
                 FROM UNNEST(onix.Subjects) as subject
-                WHERE subject.SubjectSchemeIdentifier = "BISAC_Subject_Heading") as bisac_subjects,
+                WHERE subject.SubjectSchemeIdentifier = "BISAC_Subject_Heading"
+                AND subject.SubjectCode IS NOT NULL) as bisac_subjects,
             ARRAY(
                 SELECT
                     subject.SubjectCode
                 FROM UNNEST(onix.Subjects) as subject
-                WHERE subject.SubjectSchemeIdentifier = "Thema_subject_category") as thema_subjects,
+                WHERE subject.SubjectSchemeIdentifier = "Thema_subject_category"
+                AND subject.SubjectCode IS NOT NULL) as thema_subjects,
             (SELECT 
                 custom_split(heading,';') 
             FROM UNNEST(onix.Subjects) as subject, UNNEST(subject.SubjectHeadingText) as heading