Port to Python 3.10 and cleanup (CADC-10916) (#157)

* Port to Python 3.10 and cleanup

.github/workflows/cibuild.yml

@@ -41,7 +41,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: [3.6,3.7,3.8,3.9]
+        python-version: ["3.6","3.7","3.8","3.9","3.10"]
         package: [caom2, caom2utils, caom2repo]
     steps:
     - name: Checkout code

README.rst

@@ -12,3 +12,88 @@ Common Archive Observation Model - data engineering tools
 .. image:: https://img.shields.io/github/contributors/opencadc/caom2tools.svg
     :target: https://github.com/opencadc/caom2tools/graphs/contributors
 
+
+
+
+Set of Python tools for working with the cADC CAOM2 data model: https://www.opencadc.org/caom2/
+
+
+Developers Guide
+================
+
+
+Requires pip.
+
+Installing Packages
+-------------------
+Note: might need to escape chars in your shell
+
+::
+
+    cd caom2 && pip install -e .[test]
+    cd caom2utils && pip install -e .[test]
+    cd caom2repo && pip install -e .[test]
+
+Testing packages
+----------------
+
+Testing caom2
+~~~~~~~~~~~~~~~~~
+
+::
+
+    cd ./caom2
+    pytest caom2
+
+Testing caom2utils
+~~~~~~~~~~~~~~~~
+
+::
+
+    cd ./caom2utils
+    pytest caom2utils
+
+Testing caom2rep
+~~~~~~~~~~~~~~~~
+
+::
+
+    cd ./caom2repo
+    pytest caom2repo
+
+
+
+Checkstyle
+~~~~~~~~~~
+flake8 style checking is enforced on pull requests. Following commands should
+not report errors
+
+::
+
+     flake8 caom2/caom2 caom2utils/caom2utils caom2repo/caom2repo
+
+
+Testing with tox
+~~~~~~~~~~~~~~~~
+
+If tox, the generic virtual environment tool, is available it can be used to test with different versions of
+python is isolation. For example, to test on all supported versions of Python in cadcdata (assuming that
+they are available in the system):
+
+::
+
+    cd ./caom2repo && tox
+
+To test a specific version:
+
+::
+
+    cd ./caom2utils && tox -e py3.9
+
+
+To list all the available environments:
+
+::
+
+    cd ./caom2 && tox -a
+

caom2/caom2/caom_util.py

@@ -204,7 +204,7 @@ def value_check(value, min_value, max_value, variable, override=None):
     return True
 
 
-class TypedList(collections_abc.MutableSequence):
+class TypedList(collections.abc.MutableSequence):
     """
     Class that implements a typed list in Python. Supported types
     are specified when instance is created. Example:

caom2/setup.cfg

@@ -35,13 +35,25 @@ license = AGPLv3
 url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2
 edit_on_github = False
 github_project = opencadc/caom2tools
+# version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
+version = 2.5
+
+[options]
 install_requires =
-    future aenum six deprecated
+    future
+    aenum
+    six
+    deprecated
     lxml<=4.3.0;python_version=="3.4"
     lxml>=3.7.0;python_version>="3.5"
-# version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 2.4
 
+[options.extras_require]
+test =
+    pytest>=4.6
+    pytest-cov>=2.5.1
+    flake8>=3.4.1
+    funcsigs==1.0.2
+    mock==2.0.0
 
 [entry_points]
 caom2-checksum = caom2.checksum:checksum_diff

caom2/tox.ini

@@ -5,18 +5,19 @@ name = caom2
 
 [tox]
 envlist =
-    py{27,34,35,36,37,38,39}
+    py{27,34,35,36,37,38,39,310}
 requires =
     pip >= 19.3.1
 
 [testenv]
 description = run tests
 changedir = {toxinidir}
 passenv = HOME
-deps = -rdev_requirements.txt
 commands =
     pip freeze
     pytest {[package]name} 
+extras:
+    test
 
 [testenv:egg_info]
 description = ensure egg_info works without dependencies
@@ -27,7 +28,6 @@ commands =
 description = determine the code coverage
 deps:
     coverage>=4.5.4
-    -rdev_requirements.txt
 commands =
     pytest {[package]name} --cov {[package]name} --cov-report xml --cov-config={toxinidir}/setup.cfg
 

caom2repo/caom2repo/tests/test_core.py

@@ -958,47 +958,55 @@ def test_help(self):
             sys.argv = ["caom2-repo", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(usage, stdout_mock.getvalue())
+            # Python 3.10 difference in titles
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert usage.strip('\n') == actual
 
         # create --help
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             sys.argv = ["caom2-repo", "create", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(create_usage, stdout_mock.getvalue())
-        # print(stdout_mock.getvalue())
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert create_usage.strip('\n') == actual
 
         # read --help
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             sys.argv = ["caom2-repo", "read", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(read_usage, stdout_mock.getvalue())
-        # print(stdout_mock.getvalue())
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert read_usage.strip('\n') == actual
 
         # update --help
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             sys.argv = ["caom2-repo", "update", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(update_usage, stdout_mock.getvalue())
-        # print(stdout_mock.getvalue())
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert update_usage.strip('\n') == actual
 
         # delete --help
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             sys.argv = ["caom2-repo", "delete", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(delete_usage, stdout_mock.getvalue())
-        # print(stdout_mock.getvalue())
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert delete_usage.strip('\n') == actual
 
         # visit --help
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             sys.argv = ["caom2-repo", "visit", "--help"]
             with self.assertRaises(MyExitError):
                 core.main_app()
-            self.assertEqual(visit_usage, stdout_mock.getvalue())
-        # print(stdout_mock.getvalue())
+            actual = stdout_mock.getvalue().replace(
+                'options:', 'optional arguments:').strip('\n')
+            assert visit_usage.strip('\n') == actual
 
         # visit too few number of threads
         with patch('sys.stderr', new_callable=StringIO) as stderr_mock:
@@ -1011,7 +1019,6 @@ def test_help(self):
                 core.main_app()
             self.assertTrue('error: argument --threads: invalid choice' in
                             stderr_mock.getvalue())
-        # print(stderr_mock.getvalue())
 
         # visit too many number of threads
         with patch('sys.stderr', new_callable=StringIO) as stderr_mock:
@@ -1024,4 +1031,3 @@ def test_help(self):
                 core.main_app()
             self.assertTrue('error: argument --threads: invalid choice' in
                             stderr_mock.getvalue())
-        # print(stderr_mock.getvalue())

caom2repo/setup.cfg

@@ -31,11 +31,22 @@ license = AGPLv3
 url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2
 edit_on_github = False
 github_project = opencadc/caom2tools
-install_requires = cadcutils>=1.2.3 caom2>=2.4
 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 1.5.3
-
-
+version = 1.5.4
+
+[options]
+install_requires =
+    cadcutils>=1.2.3
+    caom2>=2.5
+
+[options.extras_require]
+test =
+    pytest
+    pytest-cov>=2.5.1
+    flake8>=3.4.1
+    funcsigs==1.0.2
+    mock>=2.0.0
+    xml-compare>=1.0.5
 
 [entry_points]
 caom2-repo = caom2repo.core:main_app

caom2repo/tox.ini

@@ -5,18 +5,21 @@ name = caom2repo
 
 [tox]
 envlist =
-    py{36,37,38,39}
+    py{36,37,38,39,310}
 requires =
     pip >= 19.3.1
 
 [testenv]
 description = run tests
 changedir = {toxinidir}
 passenv = HOME
-deps = -rdev_requirements.txt
+deps =
+    -e ../caom2
 commands =
     pip freeze
     pytest {[package]name} 
+extras =
+    test
 
 [testenv:egg_info]
 description = ensure egg_info works without dependencies
@@ -27,7 +30,7 @@ commands =
 description = determine the code coverage
 deps:
     coverage>=4.5.4
-    -rdev_requirements.txt
+    -e ../caom2
 commands =
     pytest {[package]name} --cov {[package]name} --cov-report xml --cov-config={toxinidir}/setup.cfg
 
@@ -61,6 +64,7 @@ description = check code style, e.g. with flake8
 # F822: undefined name in __all__
 # F823: local variable name referenced before assignment
 skip_install = true
-deps = flake8
+deps = 
+    flake8
 changedir = {toxinidir}
 commands = flake8 {[package]name} --count --select=E101,W191,W291,W292,W293,W391,E111,E112,E113,E30,E502,E722,E901,E902,E999,F822,F823

caom2utils/caom2utils/tests/test_fits2caom2.py

@@ -542,7 +542,9 @@ def test_help():
         sys.argv = ["fits2caom2", "-h"]
         with pytest.raises(MyExitError):
             main_app()
-        assert (usage == stdout_mock.getvalue())
+        expected = stdout_mock.getvalue().replace(
+            'options:', 'optional arguments:').strip('\n')
+        assert usage.strip('\n') == expected
 
     # missing productID when plane count is wrong
     with patch('sys.stderr', new_callable=StringIO) as stderr_mock:

caom2utils/setup.cfg

@@ -33,12 +33,12 @@ url = https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2
 edit_on_github = False
 github_project = opencadc/caom2tools
 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 1.6.4
+version = 1.6.5
 
 [options]
 install_requires =
    cadcdata==2.0
-   caom2>=2.4
+   caom2>=2.5
    astropy>=2.0
    spherical-geometry>=1.2.11;python_version>="3.6"
    vos>=3.1.1

caom2utils/tox.ini

@@ -5,7 +5,7 @@ name = caom2utils
 
 [tox]
 envlist =
-    py{36,37,38,39}
+    py{36,37,38,39,310}
 requires =
     pip >= 19.3.1
 
@@ -15,10 +15,11 @@ passenv = HOME
 deps =
     -e ../caom2
     .[h5]
-    .[test]
 commands =
     pip freeze
     pytest {[package]name} 
+extras =
+    test
 
 [testenv:egg_info]
 description = ensure egg_info works without dependencies
@@ -29,6 +30,7 @@ commands =
 description = determine the code coverage
 deps:
     coverage>=4.5.4
+    ../caom2
     .[h5]
     .[test]
 commands =