CADC-12805 - support additional Range blueprint test cases, remove de…

…precated imp import (#172) * CADC-12805 - stop partial Function creation from keywords, so that Range by blueprint works.
opencadc · Nov 17, 2023 · 7aed20c · 7aed20c
1 parent 2b6727f
commit 7aed20c
Show file tree

Hide file tree

Showing 18 changed files with 960 additions and 298 deletions.
diff --git a/caom2repo/caom2repo/core.py b/caom2repo/caom2repo/core.py
@@ -67,7 +67,8 @@
 # ***********************************************************************
 #
 import argparse
-import imp
+import importlib.util
+import importlib.machinery
 import logging
 import multiprocessing
 from multiprocessing import Pool
@@ -440,9 +441,14 @@ def _load_plugin_class(self, filepath):
         mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
 
         if file_ext.lower() == '.pyc':
-            py_mod = imp.load_compiled(mod_name, filepath)
+            loader = importlib.machinery.SourcelessFileLoader(mod_name, filepath)
         else:
-            py_mod = imp.load_source(mod_name, filepath)
+            loader = importlib.machinery.SourceFileLoader(mod_name, filepath)
+        spec = importlib.util.spec_from_file_location(mod_name, filepath, loader=loader)
+        py_mod = importlib.util.module_from_spec(spec)
+        # cache the module
+        sys.modules[py_mod.__name__] = py_mod
+        spec.loader.exec_module(py_mod)
 
         if hasattr(py_mod, expected_class):
             self.plugin = getattr(py_mod, expected_class)()

diff --git a/caom2repo/setup.cfg b/caom2repo/setup.cfg
@@ -32,7 +32,7 @@ url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2
 edit_on_github = False
 github_project = opencadc/caom2tools
 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 1.6
+version = 1.6.1
 
 [options]
 install_requires =

diff --git a/caom2utils/caom2utils/caom2blueprint.py b/caom2utils/caom2utils/caom2blueprint.py
diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.blueprint b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.blueprint
@@ -0,0 +1,93 @@
+Observation.observationID = ['OBSID'], default = None
+Observation.type = object
+Observation.intent = science
+Observation.metaRelease = 2023-10-31T00:45:40.102340
+Observation.metaProducer = brite2caom2/0.1.1
+Observation.algorithm.name = exposure
+Observation.instrument.name = UniBRITE
+Observation.instrument.keywords = None
+Observation.target.name = HD37043
+Observation.target.standard = False
+Observation.telescope.geoLocationX = None
+Observation.telescope.geoLocationY = None
+Observation.telescope.geoLocationZ = None
+Observation.telescope.name = UniBRITE
+Observation.proposal.id = None
+Observation.environment.ambientTemp = None
+Plane.productID = timeseries
+Plane.metaRelease = 2023-10-31T00:45:40.102340
+Plane.dataRelease = 2023-10-31T00:45:40.102340
+Plane.dataProductType = timeseries
+Plane.calibrationLevel = 2
+Plane.metaProducer = brite2caom2/0.1.1
+Plane.provenance.name = BRITE Specific Aperture Photometry - chopping mode
+Plane.provenance.version = APa3s2chop
+Plane.provenance.project = BRITE-Constellation Nano-Satellites for Astrophysics
+Plane.provenance.producer = Adam Popowicz<Adam.Popowicz@polsl.pl>
+Plane.provenance.runID = R4
+Plane.provenance.reference = http://brite-wiki.astro.uni.wroc.pl/bwiki/doku.php?id=start
+Plane.provenance.lastExecuted = 2023-12-31T00:45:40
+Artifact.productType = _get_artifact_product_type(uri)
+Artifact.releaseType = data
+Artifact.metaProducer = brite2caom2/0.1.1
+Chunk = include
+Chunk.metaProducer = brite2caom2/0.1.1
+Chunk.position.coordsys = ICRS
+Chunk.position.equinox =  None
+Chunk.position.axis.axis1.ctype = RA---TAN
+Chunk.position.axis.axis1.cunit = deg
+Chunk.position.axis.axis2.ctype = DEC--TAN
+Chunk.position.axis.axis2.cunit = deg
+Chunk.position.axis.error1.syser = None
+Chunk.position.axis.error1.rnder = None
+Chunk.position.axis.error2.syser = None
+Chunk.position.axis.error2.rnder = None
+Chunk.position.axis.function.cd11 = 0.0375
+Chunk.position.axis.function.cd12 = 0.0
+Chunk.position.axis.function.cd21 = 0.0
+Chunk.position.axis.function.cd22 = 0.0375
+Chunk.position.axis.function.dimension.naxis1 = 1
+Chunk.position.axis.function.dimension.naxis2 = 1
+Chunk.position.axis.function.refCoord.coord1.pix = 1.0
+Chunk.position.axis.function.refCoord.coord1.val = 83.85825794708
+Chunk.position.axis.function.refCoord.coord2.pix = 1.0
+Chunk.position.axis.function.refCoord.coord2.val = -5.9099009825
+Chunk.energy.specsys = TOPOCENT
+Chunk.energy.ssysobs = None
+Chunk.energy.restfrq = None
+Chunk.energy.restwav = None
+Chunk.energy.velosys = None
+Chunk.energy.zsource = None
+Chunk.energy.ssyssrc = None
+Chunk.energy.velang = None
+Chunk.energy.bandpassName = Red
+Chunk.energy.resolvingPower = 3.8750000000000018
+Chunk.energy.axis.axis.ctype = WAVE
+Chunk.energy.axis.axis.cunit = m
+Chunk.energy.axis.error.syser = None
+Chunk.energy.axis.error.rnder = None
+Chunk.energy.axis.function.naxis = None
+Chunk.energy.axis.function.delta = None
+Chunk.energy.axis.function.refCoord.pix = None
+Chunk.energy.axis.function.refCoord.val = None
+Chunk.energy.axis.range.start.pix = 0.5
+Chunk.energy.axis.range.start.val = 5.4e-07
+Chunk.energy.axis.range.end.pix = 1.5
+Chunk.energy.axis.range.end.val = 7e-07
+Chunk.time.exposure = 1.0
+Chunk.time.resolution = None
+Chunk.time.timesys = UTC
+Chunk.time.trefpos = None
+Chunk.time.mjdref = None
+Chunk.time.axis.axis.ctype = TIME
+Chunk.time.axis.axis.cunit = d
+Chunk.time.axis.error.syser = None
+Chunk.time.axis.error.rnder = None
+Chunk.time.axis.function.naxis = None
+Chunk.time.axis.function.delta = None
+Chunk.time.axis.function.refCoord.pix = None
+Chunk.time.axis.function.refCoord.val = None
+Chunk.time.axis.range.start.pix = 0.5
+Chunk.time.axis.range.start.val = _get_time_axis_range_start_val(uri)
+Chunk.time.axis.range.end.pix = 1.5
+Chunk.time.axis.range.end.val = _get_time_axis_range_end_val(uri)
diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.module b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.module
@@ -0,0 +1,13 @@
+from caom2 import ProductType
+
+
+def _get_artifact_product_type(uri):
+    return ProductType.SCIENCE
+
+
+def _get_time_axis_range_end_val(uri):
+    return 59640.65652099997
+
+
+def _get_time_axis_range_start_val(uri):
+    return 59468.76826100005
diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.orig.header b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.orig.header
diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.py b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.py
@@ -0,0 +1,13 @@
+from caom2 import ProductType
+
+
+def _get_artifact_product_type(uri):
+    return ProductType.SCIENCE
+
+
+def _get_time_axis_range_end_val(uri):
+    return 59640.65652099997
+
+
+def _get_time_axis_range_start_val(uri):
+    return 59468.76826100005
diff --git a/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.xml b/caom2utils/caom2utils/tests/data/brite/HD36486/HD36486.xml
@@ -0,0 +1,134 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<caom2:Observation xmlns:caom2="http://www.opencadc.org/caom2/xml/v2.4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="caom2:SimpleObservation" caom2:id="2de6ebb4-f7fe-4b50-9562-dc89faa2cf8b" caom2:metaProducer="brite2caom2/0.1.0">
+  <caom2:collection>BRITE-Constellation</caom2:collection>
+  <caom2:observationID>HD36486_65-Ori-VIII-2021_BAb_1_5_A</caom2:observationID>
+  <caom2:metaRelease>2023-10-31T00:45:40.102</caom2:metaRelease>
+  <caom2:algorithm>
+    <caom2:name>exposure</caom2:name>
+  </caom2:algorithm>
+  <caom2:type>object</caom2:type>
+  <caom2:intent>science</caom2:intent>
+  <caom2:target>
+    <caom2:name>HD37043</caom2:name>
+  </caom2:target>
+  <caom2:telescope>
+    <caom2:name>UniBRITE</caom2:name>
+  </caom2:telescope>
+  <caom2:instrument>
+    <caom2:name>UniBRITE</caom2:name>
+  </caom2:instrument>
+  <caom2:planes>
+    <caom2:plane caom2:id="d29e7657-7cd2-45e8-8ae7-bb682ff9079c" caom2:metaProducer="brite2caom2/0.1.0">
+      <caom2:productID>HD36486_65-Ori-VIII-2021_BAb_1_5_A</caom2:productID>
+      <caom2:metaRelease>2023-10-31T00:45:40.102</caom2:metaRelease>
+      <caom2:dataRelease>2023-10-31T00:45:40.102</caom2:dataRelease>
+      <caom2:dataProductType>timeseries</caom2:dataProductType>
+      <caom2:calibrationLevel>2</caom2:calibrationLevel>
+      <caom2:provenance>
+        <caom2:name>BRITE Specific Aperture Photometry - chopping mode</caom2:name>
+        <caom2:version>APa3s2chop</caom2:version>
+        <caom2:project>BRITE-Constellation Nano-Satellites for Astrophysics</caom2:project>
+        <caom2:producer>Adam Popowicz&lt;Adam.Popowicz@polsl.pl&gt;</caom2:producer>
+        <caom2:runID>R4</caom2:runID>
+        <caom2:reference>http://brite-wiki.astro.uni.wroc.pl/bwiki/doku.php?id=start</caom2:reference>
+        <caom2:lastExecuted>2023-12-31T00:45:40.000</caom2:lastExecuted>
+      </caom2:provenance>
+      <caom2:artifacts>
+        <caom2:artifact caom2:id="ee92448e-a95c-4207-b271-6121c3ebc7bf" caom2:metaProducer="brite2caom2/0.1.0">
+          <caom2:uri>ad:BRITE-Constellation/HD36486.orig</caom2:uri>
+          <caom2:productType>science</caom2:productType>
+          <caom2:releaseType>data</caom2:releaseType>
+          <caom2:contentType>text/plain</caom2:contentType>
+          <caom2:contentLength>1600290</caom2:contentLength>
+          <caom2:contentChecksum>md5:4d3e7821c9ac7167349f999feec20536</caom2:contentChecksum>
+          <caom2:parts>
+            <caom2:part caom2:id="e556d08f-579d-4f27-8b55-bafbd4010ff3">
+              <caom2:name>0</caom2:name>
+              <caom2:chunks>
+                <caom2:chunk caom2:id="a539459c-9aeb-4ce4-b04c-19a036bbd2db" caom2:metaProducer="brite2caom2/0.1.0">
+                  <caom2:naxis>4</caom2:naxis>
+                  <caom2:positionAxis1>1</caom2:positionAxis1>
+                  <caom2:positionAxis2>2</caom2:positionAxis2>
+                  <caom2:position>
+                    <caom2:axis>
+                      <caom2:axis1>
+                        <caom2:ctype>RA---TAN</caom2:ctype>
+                        <caom2:cunit>deg</caom2:cunit>
+                      </caom2:axis1>
+                      <caom2:axis2>
+                        <caom2:ctype>DEC--TAN</caom2:ctype>
+                        <caom2:cunit>deg</caom2:cunit>
+                      </caom2:axis2>
+                      <caom2:function>
+                        <caom2:dimension>
+                          <caom2:naxis1>1</caom2:naxis1>
+                          <caom2:naxis2>1</caom2:naxis2>
+                        </caom2:dimension>
+                        <caom2:refCoord>
+                          <caom2:coord1>
+                            <caom2:pix>1.0</caom2:pix>
+                            <caom2:val>83.85825794708</caom2:val>
+                          </caom2:coord1>
+                          <caom2:coord2>
+                            <caom2:pix>1.0</caom2:pix>
+                            <caom2:val>-5.9099009825</caom2:val>
+                          </caom2:coord2>
+                        </caom2:refCoord>
+                        <caom2:cd11>0.0375</caom2:cd11>
+                        <caom2:cd12>0.0</caom2:cd12>
+                        <caom2:cd21>0.0</caom2:cd21>
+                        <caom2:cd22>0.0375</caom2:cd22>
+                      </caom2:function>
+                    </caom2:axis>
+                    <caom2:coordsys>ICRS</caom2:coordsys>
+                  </caom2:position>
+                  <caom2:energy>
+                    <caom2:axis>
+                      <caom2:axis>
+                        <caom2:ctype>WAVE</caom2:ctype>
+                        <caom2:cunit>m</caom2:cunit>
+                      </caom2:axis>
+                      <caom2:range>
+                        <caom2:start>
+                          <caom2:pix>0.5</caom2:pix>
+                          <caom2:val>5.4e-07</caom2:val>
+                        </caom2:start>
+                        <caom2:end>
+                          <caom2:pix>1.5</caom2:pix>
+                          <caom2:val>7e-07</caom2:val>
+                        </caom2:end>
+                      </caom2:range>
+                    </caom2:axis>
+                    <caom2:specsys>TOPOCENT</caom2:specsys>
+                    <caom2:bandpassName>Red</caom2:bandpassName>
+                    <caom2:resolvingPower>3.8750000000000018</caom2:resolvingPower>
+                  </caom2:energy>
+                  <caom2:time>
+                    <caom2:axis>
+                      <caom2:axis>
+                        <caom2:ctype>TIME</caom2:ctype>
+                        <caom2:cunit>d</caom2:cunit>
+                      </caom2:axis>
+                      <caom2:range>
+                        <caom2:start>
+                          <caom2:pix>0.5</caom2:pix>
+                          <caom2:val>59468.76826100005</caom2:val>
+                        </caom2:start>
+                        <caom2:end>
+                          <caom2:pix>1.5</caom2:pix>
+                          <caom2:val>59640.65652099997</caom2:val>
+                        </caom2:end>
+                      </caom2:range>
+                    </caom2:axis>
+                    <caom2:timesys>UTC</caom2:timesys>
+                    <caom2:exposure>1.0</caom2:exposure>
+                  </caom2:time>
+                </caom2:chunk>
+              </caom2:chunks>
+            </caom2:part>
+          </caom2:parts>
+        </caom2:artifact>
+      </caom2:artifacts>
+    </caom2:plane>
+  </caom2:planes>
+</caom2:Observation>
diff --git a/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.blueprint b/caom2utils/caom2utils/tests/data/gemini/N20030325S0098/N20030325S0098.blueprint
@@ -0,0 +1,63 @@
+Observation.observationID = ['OBSID'], default = None
+Observation.type = OBJECT
+Observation.intent = science
+Observation.metaRelease = 2003-03-25T06:01:59.700
+Observation.metaProducer = gemini2caom2/0.0.0
+Observation.algorithm.name = exposure
+Observation.instrument.name = NIRI
+Observation.instrument.keywords = ['INSTMODE'], default = None
+Observation.telescope.name = ['TELESCOP'], default = None
+Observation.telescope.geoLocationX = -5464284.386715498
+Observation.telescope.geoLocationY = -2493782.309107667
+Observation.telescope.geoLocationZ = 2150786.380689657
+Observation.environment.ambientTemp = ['TEMPERAT'], default = None
+Plane.productID = N20030325S0098
+Plane.metaRelease = 2003-03-25T06:01:59.700
+Plane.dataRelease = 2004-09-25T00:00:00.000
+Plane.dataProductType = image
+Plane.calibrationLevel = 1
+Plane.metaProducer = gemini2caom2/0.0.0
+Plane.provenance.name = Gemini Observatory Data
+Plane.provenance.project = Gemini Archive
+Plane.provenance.producer = ['ORIGIN'], default = Gemini Observatory
+Plane.provenance.reference = http://archive.gemini.edu/searchform/GN-2003A-Q-51-2-004
+Plane.provenance.lastExecuted = ['DATE-FTS'], default = None
+Artifact.productType = science
+Artifact.releaseType = data
+Artifact.uri = gemini:GEMINI/N20030325S0098.fits
+Artifact.metaProducer = gemini2caom2/0.0.0
+Chunk = include
+Chunk.metaProducer = gemini2caom2/0.0.0
+Chunk.position.coordsys = ['RADESYS'], default = None
+Chunk.position.equinox = ['EPOCH'], default = 2000.0
+Chunk.position.axis.axis1.ctype = ['CTYPE1'], default = None
+Chunk.position.axis.axis1.cunit = ['CUNIT1'], default = None
+Chunk.position.axis.axis2.ctype = ['CTYPE2'], default = None
+Chunk.position.axis.axis2.cunit = ['CUNIT2'], default = None
+Chunk.position.axis.error1.syser = ['CSYER1'], default = None
+Chunk.position.axis.error1.rnder = ['CRDER1'], default = None
+Chunk.position.axis.error2.syser = ['CSYER2'], default = None
+Chunk.position.axis.error2.rnder = ['CRDER2'], default = None
+Chunk.position.axis.function.cd11 = ['CD1_1'], default = None
+Chunk.position.axis.function.cd12 = ['CD1_2'], default = None
+Chunk.position.axis.function.cd21 = ['CD2_1'], default = None
+Chunk.position.axis.function.cd22 = ['CD2_2'], default = None
+Chunk.position.axis.function.dimension.naxis1 = ['NAXIS1'], default = None
+Chunk.position.axis.function.dimension.naxis2 = ['NAXIS2'], default = None
+Chunk.position.axis.function.refCoord.coord1.pix = ['CRPIX1'], default = None
+Chunk.position.axis.function.refCoord.coord1.val = ['CRVAL1'], default = None
+Chunk.position.axis.function.refCoord.coord2.pix = ['CRPIX2'], default = None
+Chunk.position.axis.function.refCoord.coord2.val = ['CRVAL2'], default = None
+Chunk.time.exposure = get_exposure(uri)
+Chunk.time.resolution = get_exposure(uri)
+Chunk.time.timesys = ['TIMESYS'], default = None
+Chunk.time.trefpos = ['TREFPOS'], default = None
+Chunk.time.mjdref = ['MJDREF'], default = None
+Chunk.time.axis.axis.ctype = TIME
+Chunk.time.axis.axis.cunit = d
+Chunk.time.axis.error.syser = 1e-07
+Chunk.time.axis.error.rnder = 1e-07
+Chunk.time.axis.function.naxis = 1
+Chunk.time.axis.function.delta = get_time_delta()
+Chunk.time.axis.function.refCoord.pix = 0.5
+Chunk.time.axis.function.refCoord.val = 52723.25138541667