Configure Sentry

Added sentry as a dependency.
cern-sis · Oct 22, 2024 · f3ff360 · f3ff360
1 parent 5a98322
commit f3ff360
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 7 deletions.
diff --git a/requirements-airflow.txt b/requirements-airflow.txt
@@ -1,3 +1,3 @@
 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.3/constraints-3.10.txt
-apache-airflow[celery, postgres, redis, cncf.kubernetes]==2.8.3
+apache-airflow[celery, postgres, redis, cncf.kubernetes, sentry]==2.8.3
 
diff --git a/tests/integration/iop/test_iop_dag_pull_sftp.py b/tests/integration/iop/test_iop_dag_pull_sftp.py
@@ -91,7 +91,9 @@ def test_dag_run(dag, dag_was_paused: bool, iop_empty_repo):
 
 
 def test_dag_migrate_from_FTP(iop_empty_repo):
+    iop_empty_repo.delete_all()
     assert len(iop_empty_repo.find_all()) == 0
+
     with IOPSFTPService() as sftp:
         migrate_from_ftp(
             sftp,
@@ -157,10 +159,16 @@ def test_dag_migrate_from_FTP(iop_empty_repo):
             },
             {"xml": "extracted/aca95c/aca95c.xml"},
         ]
-        for (file_from_repo, expected_file) in zip(
-            iop_empty_repo.find_all(), expected_files
-        ):
-            assert file_from_repo == expected_file
+
+        assert len(iop_empty_repo.find_all()) == len(expected_files)
+
+        iop_pdf_files = sorted(item["pdf"] for item in iop_empty_repo.find_all() if "pdf" in item)
+        expected_pdf_files = sorted(item["pdf"] for item in expected_files if "pdf" in item)
+        assert iop_pdf_files == expected_pdf_files
+
+        iop_xml_files = sorted(item["xml"] for item in iop_empty_repo.find_all() if "xml" in item)
+        expected_xml_files = sorted(item["xml"] for item in expected_files if "xml" in item)
+        assert iop_xml_files == expected_xml_files
 
 
 def test_dag_trigger_file_processing():

diff --git a/tests/integration/iop/test_repo.py b/tests/integration/iop/test_repo.py
@@ -13,6 +13,9 @@ def iop_empty_repo():
 
 
 def test_pull_from_sftp(iop_empty_repo):
+    iop_empty_repo.delete_all()
+    assert len(iop_empty_repo.find_all()) == 0
+
     with IOPSFTPService() as sftp:
         migrate_from_ftp(
             sftp,
@@ -30,6 +33,7 @@ def test_pull_from_sftp(iop_empty_repo):
                 }
             }
         )
+
         expected_files = [
             {
                 "pdf": "extracted/2022-07-30T03_02_01_content/1674-1137/1674-1137_46/1674-1137_46_8/1674-1137_46_8_085001/cpc_46_8_085001.pdf",
@@ -77,7 +81,17 @@ def test_pull_from_sftp(iop_empty_repo):
             {"xml": "extracted/aca95c/aca95c.xml"},
 
         ]
-        assert iop_empty_repo.find_all() == expected_files
+
+        assert len(iop_empty_repo.find_all()) == len(expected_files)
+
+        iop_pdf_files = sorted(item["pdf"] for item in iop_empty_repo.find_all() if "pdf" in item)
+        expected_pdf_files = sorted(item["pdf"] for item in expected_files if "pdf" in item)
+        assert iop_pdf_files == expected_pdf_files
+
+        iop_xml_files = sorted(item["xml"] for item in iop_empty_repo.find_all() if "xml" in item)
+        expected_xml_files = sorted(item["xml"] for item in expected_files if "xml" in item)
+        assert iop_xml_files == expected_xml_files
+
         assert sorted(iop_empty_repo.get_all_raw_filenames()) == sorted(
             [
                 "2022-07-30T03_02_01_content.zip",
@@ -86,4 +100,4 @@ def test_pull_from_sftp(iop_empty_repo):
                 "2022-09-24T03_01_43_content.zip",
                 "aca95c.zip",
             ]
-        )
+        )