From 2e1fca7ff980ff29721dfda6677f11883f193c73 Mon Sep 17 00:00:00 2001 From: andamian Date: Fri, 18 Jan 2019 13:20:21 -0800 Subject: [PATCH] Changed the default resource ID (#102) * Changed default service for caom2repo to ivo://cadc.nrc.ca/ams --- caom2pipe/caom2pipe/execute_composable.py | 29 ++++++++++++------- caom2pipe/caom2pipe/manage_composable.py | 5 ++-- .../tests/test_execute_composable.py | 3 +- .../caom2pipe/tests/test_manage_composable.py | 9 +++--- caom2repo/caom2repo/core.py | 2 +- .../caom2repo/tests/data/create_help.txt | 3 +- .../caom2repo/tests/data/delete_help.txt | 3 +- caom2repo/caom2repo/tests/data/read_help.txt | 3 +- .../caom2repo/tests/data/update_help.txt | 3 +- caom2repo/caom2repo/tests/data/visit_help.txt | 3 +- caom2repo/setup.cfg | 4 +-- caom2utils/caom2utils/fits2caom2.py | 10 +++---- .../caom2utils/tests/data/text.override | 2 ++ .../caom2utils/tests/test_fits2caom2.py | 8 +++-- 14 files changed, 45 insertions(+), 42 deletions(-) diff --git a/caom2pipe/caom2pipe/execute_composable.py b/caom2pipe/caom2pipe/execute_composable.py index d1d9cc4b..914fd3d2 100644 --- a/caom2pipe/caom2pipe/execute_composable.py +++ b/caom2pipe/caom2pipe/execute_composable.py @@ -387,7 +387,9 @@ def _fits2caom2_cmd_client(self): mc.exec_cmd(cmd) def _fits2caom2_cmd_client_local(self): - """Execute fits2caom with a --cert parameter and a --local parameter.""" + """ + Execute fits2caom with a --cert parameter and a --local parameter. + """ plugin = self._find_fits2caom2_plugin() # so far, the plugin is also the module :) local_fqn = os.path.join(self.working_dir, self.fname) @@ -417,9 +419,9 @@ def _fits2caom2_cmd_in_out_local_client(self): local_fqn = os.path.join(self.working_dir, self.fname) cmd = '{} {} {} --in {} --out {} --local {} ' \ '--plugin {} --module {} --lineage {}/{}'.format( - self.command_name, self.logging_level_param, self.cred_param, - self.model_fqn, self.model_fqn, local_fqn, plugin, plugin, - self.product_id, self.uri) + self.command_name, self.logging_level_param, self.cred_param, + self.model_fqn, self.model_fqn, local_fqn, plugin, plugin, + self.product_id, self.uri) mc.exec_cmd(cmd) def _compare_checksums_client(self, fname): @@ -1275,14 +1277,16 @@ def choose(self, storage_name, command_name, meta_visitors, data_visitors): self.chooser.needs_delete(observation)): executors.append( Collection2CaomLocalMetaDeleteCreateClient( - self.config, storage_name, command_name, + self.config, storage_name, + command_name, cred_param, cadc_data_client, caom_repo_client, observation, meta_visitors)) else: executors.append( Collection2CaomLocalMetaUpdateClient( - self.config, storage_name, command_name, + self.config, storage_name, + command_name, cred_param, cadc_data_client, caom_repo_client, observation, meta_visitors)) @@ -1291,15 +1295,18 @@ def choose(self, storage_name, command_name, meta_visitors, data_visitors): self.chooser.needs_delete(observation)): executors.append( Collection2CaomMetaDeleteCreateClient( - self.config, storage_name, command_name, + self.config, storage_name, + command_name, cred_param, cadc_data_client, caom_repo_client, observation, meta_visitors)) else: - executors.append(Collection2CaomMetaUpdateClient( - self.config, storage_name, command_name, - cred_param, cadc_data_client, caom_repo_client, - observation, meta_visitors)) + executors.append( + Collection2CaomMetaUpdateClient( + self.config, storage_name, + command_name, cred_param, + cadc_data_client, caom_repo_client, + observation, meta_visitors)) elif task_type == mc.TaskType.MODIFY: if self.config.use_local_files: if (executors is not None and len(executors) > 0 and diff --git a/caom2pipe/caom2pipe/manage_composable.py b/caom2pipe/caom2pipe/manage_composable.py index fd5637ee..5bacecc5 100644 --- a/caom2pipe/caom2pipe/manage_composable.py +++ b/caom2pipe/caom2pipe/manage_composable.py @@ -82,7 +82,7 @@ from cadcutils import net from cadcdata import CadcDataClient -from caom2 import ObservationWriter, ObservationReader, Artifact, ReleaseType +from caom2 import ObservationWriter, ObservationReader, Artifact from caom2 import ChecksumURI @@ -583,7 +583,6 @@ def update_for_retry(self, count): logging.info('Retry work file is {}'.format(self.work_fqn)) - @staticmethod def load_config(config_fqn): """Read a configuration as a YAML file. @@ -892,7 +891,7 @@ def write_to_file(fqn, content): try: with open(fqn, 'w') as f: f.write(content) - except Exception as e: + except Exception: logging.error('Could not write file {}'.format(fqn)) raise CadcException('Could not write file {}'.format(fqn)) diff --git a/caom2pipe/caom2pipe/tests/test_execute_composable.py b/caom2pipe/caom2pipe/tests/test_execute_composable.py index 8eae5c81..2c3b6ada 100644 --- a/caom2pipe/caom2pipe/tests/test_execute_composable.py +++ b/caom2pipe/caom2pipe/tests/test_execute_composable.py @@ -93,6 +93,7 @@ class MyExitError(Exception): pass + if six.PY3: class TestVisit: @staticmethod @@ -105,7 +106,6 @@ def visit(observation, **kwargs): assert z is not None, 'log file directory' assert observation is not None, 'undefined observation' - class TestStorageName(ec.StorageName): def __init__(self, obs_id=None, file_name=None): super(TestStorageName, self).__init__( @@ -114,7 +114,6 @@ def __init__(self, obs_id=None, file_name=None): def is_valid(self): return True - class TestChooser(ec.OrganizeChooser): def __init(self): super(TestChooser, self).__init__() diff --git a/caom2pipe/caom2pipe/tests/test_manage_composable.py b/caom2pipe/caom2pipe/tests/test_manage_composable.py index ec4791a0..3812a82f 100644 --- a/caom2pipe/caom2pipe/tests/test_manage_composable.py +++ b/caom2pipe/caom2pipe/tests/test_manage_composable.py @@ -235,13 +235,13 @@ def test_get_artifact_metadata(): # create action result = mc.get_artifact_metadata(test_fqn, ProductType.WEIGHT, - ReleaseType.META, uri=test_uri) + ReleaseType.META, uri=test_uri) assert result is not None, 'expect a result' assert isinstance(result, Artifact), 'expect an artifact' assert result.product_type == ProductType.WEIGHT, 'wrong product type' assert result.content_length == 255, 'wrong length' assert result.content_checksum.uri == \ - 'md5:c649725745805d41fc1b601e85400e60', 'wrong checksum' + 'md5:c649725745805d41fc1b601e85400e60', 'wrong checksum' # update action result.content_checksum = ChecksumURI('md5:abc') @@ -250,7 +250,7 @@ def test_get_artifact_metadata(): assert result is not None, 'expect a result' assert isinstance(result, Artifact), 'expect an artifact' assert result.content_checksum.uri == \ - 'md5:c649725745805d41fc1b601e85400e60', 'wrong checksum' + 'md5:c649725745805d41fc1b601e85400e60', 'wrong checksum' # TODO understand python mocking .... :( @@ -274,7 +274,8 @@ def test_get_artifact_metadata(): # # file_chunks = [file_content[i:i + 5].encode() # # for i in range(0, len(file_content), 5)] # # response = Mock() -# # response.headers.get.return_value = 'filename={}.gz'.format('TEST.fits') +# # response.headers.get.return_value = +# 'filename={}.gz'.format('TEST.fits') # # response.raw.read.side_effect = file_chunks # # response.history = [] # # response.status_code = 200 diff --git a/caom2repo/caom2repo/core.py b/caom2repo/caom2repo/core.py index 8b0d716a..1b4a1fb5 100755 --- a/caom2repo/caom2repo/core.py +++ b/caom2repo/caom2repo/core.py @@ -99,7 +99,7 @@ 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1' # resource ID for info -DEFAULT_RESOURCE_ID = 'ivo://cadc.nrc.ca/caom2repo' +DEFAULT_RESOURCE_ID = 'ivo://cadc.nrc.ca/ams' APP_NAME = 'caom2repo' diff --git a/caom2repo/caom2repo/tests/data/create_help.txt b/caom2repo/caom2repo/tests/data/create_help.txt index 60196b38..bf653820 100644 --- a/caom2repo/caom2repo/tests/data/create_help.txt +++ b/caom2repo/caom2repo/tests/data/create_help.txt @@ -21,8 +21,7 @@ optional arguments: netrc file to use for authentication -q, --quiet run quietly --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) + resource identifier (default ivo://cadc.nrc.ca/ams) -u, --user USER name of user to authenticate. Note: application prompts for the corresponding password! -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/delete_help.txt b/caom2repo/caom2repo/tests/data/delete_help.txt index 2e907d6b..22b02516 100644 --- a/caom2repo/caom2repo/tests/data/delete_help.txt +++ b/caom2repo/caom2repo/tests/data/delete_help.txt @@ -22,8 +22,7 @@ optional arguments: netrc file to use for authentication -q, --quiet run quietly --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) + resource identifier (default ivo://cadc.nrc.ca/ams) -u, --user USER name of user to authenticate. Note: application prompts for the corresponding password! -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/read_help.txt b/caom2repo/caom2repo/tests/data/read_help.txt index 71d7fdb9..c7ee0878 100644 --- a/caom2repo/caom2repo/tests/data/read_help.txt +++ b/caom2repo/caom2repo/tests/data/read_help.txt @@ -23,8 +23,7 @@ optional arguments: --output, -o OUTPUT destination file -q, --quiet run quietly --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) + resource identifier (default ivo://cadc.nrc.ca/ams) -u, --user USER name of user to authenticate. Note: application prompts for the corresponding password! -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/update_help.txt b/caom2repo/caom2repo/tests/data/update_help.txt index 82f71653..f65a4a0a 100644 --- a/caom2repo/caom2repo/tests/data/update_help.txt +++ b/caom2repo/caom2repo/tests/data/update_help.txt @@ -21,8 +21,7 @@ optional arguments: netrc file to use for authentication -q, --quiet run quietly --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) + resource identifier (default ivo://cadc.nrc.ca/ams) -u, --user USER name of user to authenticate. Note: application prompts for the corresponding password! -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/visit_help.txt b/caom2repo/caom2repo/tests/data/visit_help.txt index 1b81f548..c6e829f7 100644 --- a/caom2repo/caom2repo/tests/data/visit_help.txt +++ b/caom2repo/caom2repo/tests/data/visit_help.txt @@ -29,8 +29,7 @@ optional arguments: --plugin PLUGIN plugin class to update each observation -q, --quiet run quietly --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) + resource identifier (default ivo://cadc.nrc.ca/ams) --start START earliest observation to visit (UTC IVOA format: YYYY- mm-ddTH:M:S) --threads {2,3,4,5,6,7,8,9} diff --git a/caom2repo/setup.cfg b/caom2repo/setup.cfg index 3ebff0f7..c53bfd6e 100644 --- a/caom2repo/setup.cfg +++ b/caom2repo/setup.cfg @@ -33,9 +33,9 @@ license = AGPLv3 url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2 edit_on_github = False github_project = opencadc/caom2tools -install_requires = cadcutils>=1.1.9 caom2>=2.3.6 +install_requires = cadcutils>=1.1.11 caom2>=2.3.6 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -version = 1.4.2 +version = 1.4.3 diff --git a/caom2utils/caom2utils/fits2caom2.py b/caom2utils/caom2utils/fits2caom2.py index 7410f75b..5e6360ee 100755 --- a/caom2utils/caom2utils/fits2caom2.py +++ b/caom2utils/caom2utils/fits2caom2.py @@ -1558,9 +1558,6 @@ def _to_enum_type(self, value, to_enum_type): def _apply_blueprint_to_generic(self): - # pointers that are short to type - exts = self.blueprint._extensions - wcs_std = self.blueprint._wcs_std plan = self.blueprint._plan # first apply the functions @@ -2431,7 +2428,8 @@ def _get_proposal(self, current): if current is None: prop_id = self._get_from_list('Observation.proposal.id', index=0) pi = self._get_from_list('Observation.proposal.pi', index=0) - project = self._get_from_list('Observation.proposal.project', index=0) + project = self._get_from_list('Observation.proposal.project', + index=0) title = self._get_from_list('Observation.proposal.title', index=0) else: prop_id = self._get_from_list('Observation.proposal.id', index=0, @@ -2526,8 +2524,8 @@ def _get_telescope(self, current): geo_z = _to_float( self._get_from_list('Observation.telescope.geoLocationZ', index=0, current=current.geo_location_z)) - keywords = self._get_set_from_list('Observation.telescope.keywords', - index=0) # TODO + keywords = self._get_set_from_list( + 'Observation.telescope.keywords', index=0) # TODO if keywords is None: keywords = current.keywords self.logger.debug('End Telescope augmentation.') diff --git a/caom2utils/caom2utils/tests/data/text.override b/caom2utils/caom2utils/tests/data/text.override index 390f31f9..1d23142a 100644 --- a/caom2utils/caom2utils/tests/data/text.override +++ b/caom2utils/caom2utils/tests/data/text.override @@ -1 +1,3 @@ artifact.productType = thumbnail +plane.dataProductType = image +plane.calibrationLevel = 3 diff --git a/caom2utils/caom2utils/tests/test_fits2caom2.py b/caom2utils/caom2utils/tests/test_fits2caom2.py index 55b2e4a8..d6f4f8fb 100755 --- a/caom2utils/caom2utils/tests/test_fits2caom2.py +++ b/caom2utils/caom2utils/tests/test_fits2caom2.py @@ -81,6 +81,7 @@ from caom2 import get_differences, obs_reader_writer, ObservationReader, Chunk from caom2 import SpectralWCS, TemporalWCS, PolarizationWCS, SpatialWCS from caom2 import Axis, CoordAxis1D, CoordAxis2D +import logging import caom2utils @@ -946,6 +947,8 @@ def _get_obs(from_xml_string): test_product_id + image + 3 thumbnail @@ -1248,16 +1251,15 @@ def test_get_vos_meta(): @pytest.mark.skipif(single_test, reason='Single test mode') -def test_generic_parser(): +def test_generic_parser1(): test_key = 'Plane.metaRelease' test_value = '2013-10-10' test_blueprint = ObsBlueprint() test_blueprint.set(test_key, '2013-10-10') - import logging logging.error(test_blueprint) test_parser = GenericParser() assert test_parser._blueprint._plan[test_key] == \ - (['RELEASE', 'REL_DATE'], None), 'default value changed' + (['RELEASE', 'REL_DATE'], None), 'default value changed' test_parser.blueprint = test_blueprint assert test_parser._blueprint._plan[test_key] == test_value, \ 'original value over-ridden'