Merge pull request ckan#208 from JVickery-TBS/feature/group-org-dump-…

…load-users Argument to Inlude Group/Org Users in Dump and Load (+ fixed tests)
JVickery-TBS · Aug 25, 2023 · 4aa7d25 · 4aa7d25
2 parents bda19c1 + fded3ce
commit 4aa7d25
Show file tree

Hide file tree

Showing 9 changed files with 73 additions and 40 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,16 +5,16 @@ jobs:
     strategy:
       matrix:
         python-version: ["2.7", "3.8"]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+    container:
+      # INFO: python 2 is no longer supported in
+      # actions/setup-python, use python docker image instead
+      image: python:${{ matrix.python-version }}
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install requirements
+    - name: Install requirements (py ${{ matrix.python-version }})
       run: |
         pip install -e ".[testing]"
-    - name: Run all tests
+    - name: Run all tests (py ${{ matrix.python-version }})
       run: python -m unittest discover
diff --git a/ckanapi/cli/dump.py b/ckanapi/cli/dump.py
@@ -117,6 +117,8 @@ def dump_things(ckan, thing, arguments,
                     jsonl_output.write(compact_json(record,
                         sort_keys=True) + b'\n')
                 expecting_number += 1
+    if jsonl_output != stdout:
+        jsonl_output.close()
     if 'pipe' in errors:
         return 1
     if 'interrupt' in errors:
@@ -173,9 +175,14 @@ def reply(error, record=None):
             requests_kwargs = None
             if arguments['--insecure']:
                 requests_kwargs = {'verify': False}
+            include_users = False
+            if '--include-users' in arguments \
+            and arguments['--include-users']:
+                include_users = True
             obj = ckan.call_action(thing_show, {'id': name,
                 'include_datasets': False,
                 'include_password_hash': True,
+                'include_users': include_users,
                 }, requests_kwargs=requests_kwargs)
         except NotFound:
             reply('NotFound')
@@ -210,6 +217,7 @@ def b(name):
         + b('--get-request')
         + b('--datastore-fields')
         + b('--resource-views')
+        + b('--include-users')
         + ['value-here-to-make-docopt-happy']
         )
 

diff --git a/ckanapi/cli/load.py b/ckanapi/cli/load.py
@@ -185,6 +185,7 @@ def reply(action, error, response):
                             {'id': name,
                              'include_datasets': False,
                              'include_password_hash': True,
+                             'include_users': True,
                             },
                             requests_kwargs=requests_kwargs)
                     except NotFound:

diff --git a/ckanapi/cli/main.py b/ckanapi/cli/main.py
@@ -14,15 +14,15 @@
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
   ckanapi dump (datasets | groups | organizations | users | related)
           (ID_OR_NAME ... | --all) ([-O JSONL_OUTPUT] | [-D DIRECTORY])
-          [-p PROCESSES] [-dqwzR]
+          [-p PROCESSES] [-dqwzRU]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [-g] [--insecure]]
   ckanapi load datasets
           [--upload-resources] [-I JSONL_INPUT] [-s START] [-m MAX]
           [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
   ckanapi load (groups | organizations)
           [--upload-logo] [-I JSONL_INPUT] [-s START] [-m MAX]
-          [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwz]
+          [-p PROCESSES] [-l LOG_FILE] [-n | -o] [-qwzU]
           [[-c CONFIG] [-u USER] | -r SITE_URL [-a APIKEY] [--insecure]]
   ckanapi load (users | related)
           [-I JSONL_INPUT] [-s START] [-m MAX] [-p PROCESSES] [-l LOG_FILE]
@@ -70,6 +70,7 @@
                             record is number 1 [default: 1]
   -u --ckan-user=USER       perform actions as user with this name, uses the
                             site sysadmin user when not specified
+  -U --include-users        include users of a group/organization
   --upload-logo             upload logo image of a group/organization if the
                             image is stored in the original server, otherwise
                             its image url will be used

diff --git a/ckanapi/cli/search.py b/ckanapi/cli/search.py
@@ -79,3 +79,6 @@ def search_datasets(ckan, arguments, stdin=None, stdout=None, stderr=None):
             break
 
         start += len(rows)
+
+    if jsonl_output != stdout:
+        jsonl_output.close()
diff --git a/ckanapi/tests/mock/mock_ckan.py b/ckanapi/tests/mock/mock_ckan.py
@@ -3,6 +3,11 @@
 import csv
 from wsgiref.util import setup_testing_defaults
 from wsgiref.simple_server import make_server
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 
 def mock_ckan(environ, start_response):
     status = '200 OK'
@@ -39,7 +44,14 @@ def mock_ckan(environ, start_response):
             environ=environ,
             keep_blank_values=True,
             )
-        records = list(csv.reader(fs['upload'].file))
+        upload_data = fs.getvalue('upload').decode('utf-8').splitlines()
+        csv_file = StringIO()
+        writer = csv.writer(csv_file)
+        for line_data in upload_data:
+            row_data = line_data.split(',')
+            writer.writerow(row_data)
+        csv_file.seek(0)
+        records = list(csv.reader(csv_file))
         start_response(status, headers)
         return [json.dumps({
             "help": "none",

diff --git a/ckanapi/tests/test_cli_dump.py b/ckanapi/tests/test_cli_dump.py
@@ -146,7 +146,8 @@ def test_parent_dump_all(self):
                 '--get-request': False,
                 '--datastore-fields': False,
                 '--resource-views': False,
-                '--insecure': False
+                '--insecure': False,
+                '--include-users': False,
             },
             worker_pool=self._mock_worker_pool,
             stdout=self.stdout,
@@ -177,7 +178,8 @@ def test_parent_parallel_limit(self):
                 '--get-request': False,
                 '--datastore-fields': False,
                 '--resource-views': False,
-                '--insecure': False
+                '--insecure': False,
+                '--include-users': False,
             },
             worker_pool=self._mock_worker_pool,
             stdout=self.stdout,
@@ -205,7 +207,8 @@ def test_parent_id_argument(self):
                 '--get-request': False,
                 '--datastore-fields': False,
                 '--resource-views': False,
-                '--insecure': False
+                '--insecure': False,
+                '--include-users': False,
             },
 
             worker_pool=self._mock_worker_pool,
@@ -235,7 +238,8 @@ def test_parent_maintain_order(self):
                 '--get-request': False,
                 '--datastore-fields': False,
                 '--resource-views': False,
-                '--insecure': False
+                '--insecure': False,
+                '--include-users': False,
             },
             worker_pool=self._mock_worker_pool_reversed,
             stdout=self.stdout,
@@ -269,7 +273,8 @@ def test_parent_datapackages(self):
                     '--get-request': False,
                     '--datastore-fields': False,
                     '--resource-views': False,
-                    '--insecure': False
+                    '--insecure': False,
+                    '--include-users': False,
                 },
                 worker_pool=self._worker_pool_with_data,
                 stdout=self.stdout,
@@ -301,7 +306,7 @@ def test_parent_datapackages(self):
         finally:
             shutil.rmtree(target)
 
-    
+
     def test_resource_views(self):
         target = tempfile.mkdtemp()
         try:
@@ -322,7 +327,8 @@ def test_resource_views(self):
                     '--get-request': False,
                     '--datastore-fields': False,
                     '--resource-views': True,
-                    '--insecure': False
+                    '--insecure': False,
+                    '--include-users': False,
                 },
                 worker_pool=self._worker_pool_with_resource_views,
                 stdout=self.stdout,
@@ -374,20 +380,22 @@ def _worker_pool_with_data(self, cmd, processes, job_iter):
         dump_things_worker(self.ckan, 'datasets', {
             '--datastore-fields': True,
             '--resource-views': False,
-            '--insecure': False},
+            '--insecure': False,
+            '--include-users': False,},
             stdin=worker_stdin,
             stdout=worker_stdout)
         for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):
             yield [[], i, v]
 
-    
+
     def _worker_pool_with_resource_views(self, cmd, proccesses, job_iter):
         worker_stdin = BytesIO(b''.join(v for i, v in job_iter))
         worker_stdout = BytesIO()
         dump_things_worker(self.ckan, 'datasets', {
             '--datastore-fields': False,
             '--resource-views': True,
-            '--insecure': False},
+            '--insecure': False,
+            '--include-users': False,},
             stdin=worker_stdin,
             stdout=worker_stdout)
         for i, v in enumerate(worker_stdout.getvalue().strip().split(b'\n')):

diff --git a/ckanapi/tests/test_cli_load.py b/ckanapi/tests/test_cli_load.py
@@ -68,7 +68,7 @@ def test_create_with_no_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -84,7 +84,7 @@ def test_create_with_corrupted_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five","resources":[{"id":"123"}]}\n'),
             stdout=self.stdout)
@@ -100,7 +100,7 @@ def test_create_with_complete_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(
                  b'{"name": "45","title":"Forty-five",'
@@ -118,7 +118,7 @@ def test_create_only(self):
                 '--create-only': True,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -134,7 +134,7 @@ def test_create_empty_dict(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{}\n'),
             stdout=self.stdout)
@@ -149,7 +149,7 @@ def test_create_bad_option(self):
         load_things_worker(self.ckan, 'datasets', {
                 '--create-only': False,
                 '--update-only': True,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "45","title":"Forty-five"}\n'),
             stdout=self.stdout)
@@ -164,7 +164,7 @@ def test_update_with_no_resources(self):
         load_things_worker(self.ckan, 'datasets', {
                 '--create-only': False,
                 '--update-only': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -180,7 +180,7 @@ def test_update_with_corrupted_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "30ish","title":"3.4 times ten","resources":[{"id":"123"}]}\n'),
             stdout=self.stdout)
@@ -196,7 +196,7 @@ def test_update_with_complete_resources(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(
                  b'{"name": "30ish","title":"3.4 times ten",'
@@ -214,7 +214,7 @@ def test_update_only(self):
                 '--create-only': False,
                 '--update-only': True,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -230,7 +230,7 @@ def test_update_bad_option(self):
                 '--create-only': True,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "34","title":"3.4 times ten"}\n'),
             stdout=self.stdout)
@@ -246,7 +246,7 @@ def test_update_unauthorized(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"name": "seekrit", "title": "Things"}\n'),
             stdout=self.stdout)
@@ -262,7 +262,7 @@ def test_update_group(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"id": "ab","title":"a balloon"}\n'),
             stdout=self.stdout)
@@ -278,7 +278,7 @@ def test_update_organization_two(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(
                 b'{"name": "cd", "title": "Go"}\n'
@@ -302,7 +302,7 @@ def test_update_organization_with_users_unchanged(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"id": "used", "title": "here"}\n'),
             stdout=self.stdout)
@@ -318,7 +318,7 @@ def test_update_organization_with_users_cleared(self):
                 '--create-only': False,
                 '--update-only': False,
                 '--upload-resources': False,
-                '--insecure': False
+                '--insecure': False,
                 },
             stdin=BytesIO(b'{"id": "unused", "users": []}\n'),
             stdout=self.stdout)
@@ -347,7 +347,7 @@ def test_parent_load_two(self):
                 '--max-records': None,
                 '--upload-resources': False,
                 '--upload-logo': False,
-                '--insecure': False
+                '--insecure': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(
@@ -382,7 +382,7 @@ def test_parent_load_start_max(self):
                 '--max-records': '2',
                 '--upload-resources': False,
                 '--upload-logo': False,
-                '--insecure': False
+                '--insecure': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(
@@ -420,7 +420,7 @@ def test_parent_parallel_limit(self):
                 '--max-records': None,
                 '--upload-resources': False,
                 '--upload-logo': False,
-                '--insecure': False
+                '--insecure': False,
             },
             worker_pool=self._mock_worker_pool,
             stdin=BytesIO(

diff --git a/ckanapi/tests/test_remote.py b/ckanapi/tests/test_remote.py
@@ -34,7 +34,7 @@ class TestRemoteAction(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         script = os.path.join(os.path.dirname(__file__), 'mock/mock_ckan.py')
-        _mock_ckan = subprocess.Popen(['python2', script],
+        _mock_ckan = subprocess.Popen(['python', script],
             stdout=DEVNULL, stderr=DEVNULL)
         def kill_child():
             try: