refactor: make types py2 compatible

.github/workflows/test.yml

@@ -2,17 +2,22 @@ name: Tests
 on: [push, pull_request]
 jobs:
   test:
+    strategy:
+      matrix:
+        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8]
+      fail-fast: false
+
     runs-on: ubuntu-latest
     container:
       # The CKAN version tag of the Solr and Postgres containers should match
       # the one of the container the tests run on.
       # You can switch this base image with a custom image tailored to your project
-      image: openknowledge/ckan-dev:2.9
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
     services:
       solr:
-        image: ckan/ckan-solr-dev:2.9
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}
       postgres:
-        image: ckan/ckan-postgres-dev:2.9
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -33,16 +38,19 @@ jobs:
     - name: Install requirements
       # Install any extra requirements your extension has here (dev requirements, other extensions etc)
       run: |
-        pip install -r requirements.txt
-        pip install -r dev-requirements.txt
-        pip install -e .
+        pip install -e ".[dev]"
     - name: Setup extension
       # Extra initialization steps
       run: |
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.8' }}
+      run: |
         ckan -c test.ini db init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
     - name: Run tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.files --disable-warnings ckanext/files
-
+      run: pytest --ckan-ini=test.ini --cov=ckanext.files --disable-warnings ckanext/files -p no:rerunfailures

ckanext/file_manager/collection.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-from typing import Any
 from ckanext.collection.utils.data.model import ModelData
 
 from dominate import tags
 
 import ckan.plugins.toolkit as tk
 
-from ckanext.collection.types import InputFilter, LinkFilter, ButtonFilter
+from ckanext.collection.types import InputFilter, ButtonFilter
 from ckanext.collection.utils import Filters
 
 from ckanext.ap_main.collection.base import (

ckanext/files/cli.py

@@ -17,13 +17,12 @@ def files():
 @files.command()
 @click.option("--delete", "-d", is_flag=True, help="Delete orphaned datasets.")
 @click.argument("threshold", required=False, type=int)
-def remove_unused_files(delete: bool, threshold: int):
+def remove_unused_files(delete, threshold):
+    # type: (bool, int | None) -> None
     """Remove files that are not used for N days. The unused threshold is specified
     in a config"""
     threshold = (
-        threshold
-        if threshold is not None
-        else files_conf.get_unused_threshold()
+        threshold if threshold is not None else files_conf.get_unused_threshold()
     )
 
     files = tk.get_action("files_get_unused_files")(

ckanext/files/config.py

@@ -3,5 +3,6 @@
 CONF_UNUSED_THRESHOLD = "ckanext.files.unused_threshold"
 
 
-def get_unused_threshold() -> int:
-    return tk.config[CONF_UNUSED_THRESHOLD]
+def get_unused_threshold():
+    # type: () -> int
+    return tk.asint(tk.config.get(CONF_UNUSED_THRESHOLD))

ckanext/files/helpers.py

@@ -1,5 +1,6 @@
 import ckanext.files.config as files_conf
 
 
-def files_get_unused_threshold() -> int:
+def files_get_unused_threshold():
+    # type: () -> int
     return files_conf.get_unused_threshold()

ckanext/files/logic/action.py

@@ -1,7 +1,6 @@
-from __future__ import annotations
+import six
 import datetime
 import os
-from typing import Any, Optional
 
 import ckan.plugins.toolkit as tk
 from ckan.logic import validate
@@ -11,6 +10,9 @@
 from ckanext.files.model import File
 from . import schema
 
+if six.PY3:
+    from typing import Any
+
 _actions, action = make_collector()
 
 CONFIG_SIZE = "ckanext.files.kind.{kind}.max_size"
@@ -21,14 +23,15 @@ def get_actions():
     return dict(_actions)
 
 
-def files_uploader(kind: str, old_filename: Optional[str] = None):
+def files_uploader(kind, old_filename=None):
+    # type: (str, str | None) -> Any
     return get_uploader(kind, old_filename)
 
 
 @action
 @validate(schema.file_create)
 def files_file_create(context, data_dict):
-    # type: (Context, dict[str, Any]) -> dict[str, Any]
+    # type: (Any, dict[str, Any]) -> dict[str, Any]
     tk.check_access("files_file_create", context, data_dict)
 
     _upload(data_dict, data_dict["kind"])
@@ -39,13 +42,13 @@ def files_file_create(context, data_dict):
     return file.dictize(context)
 
 
-def _upload(data_dict: dict[str, Any], kind: str):
+def _upload(data_dict, kind):
+    # type: (dict[str, Any], str) -> None
+
     uploader = files_uploader(kind)
     uploader.update_data_dict(data_dict, "path", "upload", None)
 
-    max_size = tk.asint(
-        tk.config.get(CONFIG_SIZE.format(kind=kind), DEFAULT_SIZE)
-    )
+    max_size = tk.asint(tk.config.get(CONFIG_SIZE.format(kind=kind), DEFAULT_SIZE))
     uploader.upload(max_size)
 
     # TODO: try not to rely on hardcoded segments
@@ -57,12 +60,11 @@ def _upload(data_dict: dict[str, Any], kind: str):
 @action
 @validate(schema.file_update)
 def files_file_update(context, data_dict):
+    # type: (Any, dict[str, Any]) -> dict[str, Any]
+
     tk.check_access("files_file_delete", context, data_dict)
     file: File = (
-        context["session"]
-        .query(File)
-        .filter_by(id=data_dict["id"])
-        .one_or_none()
+        context["session"].query(File).filter_by(id=data_dict["id"]).one_or_none()
     )
     if not file:
         raise tk.ObjectNotFound("File not found")
@@ -79,6 +81,7 @@ def files_file_update(context, data_dict):
 
 @action
 def files_file_delete(context, data_dict):
+    # type: (Any, dict[str, Any]) -> bool
     id_ = tk.get_or_bust(data_dict, "id")
     tk.check_access("files_file_delete", context, data_dict)
     file = context["session"].query(File).filter_by(id=id_).one_or_none()
@@ -95,6 +98,7 @@ def files_file_delete(context, data_dict):
 
 @action
 def files_file_show(context, data_dict):
+    # type: (Any, dict[str, Any]) -> dict[str, Any]
     id_ = tk.get_or_bust(data_dict, "id")
     tk.check_access("files_file_show", context, data_dict)
 
@@ -111,7 +115,8 @@ def files_file_show(context, data_dict):
     return file.dictize(context)
 
 
-def _remove_file_from_filesystem(file_path: str) -> bool:
+def _remove_file_from_filesystem(file_path):
+    # type: (str) -> bool
     """Remove a file from the file system"""
     storage_path = get_storage_path()
     file_path = os.path.join(storage_path, "storage", file_path)
@@ -134,6 +139,7 @@ def _remove_file_from_filesystem(file_path: str) -> bool:
 @tk.side_effect_free
 @validate(schema.file_get_unused_files)
 def files_get_unused_files(context, data_dict):
+    # type: (Any, dict[str, Any]) -> list[dict[str, Any]]
     """Return a list of unused file based on a configured threshold"""
     tk.check_access("files_get_unused_files", context, data_dict)
 

ckanext/files/logic/auth.py

@@ -1,5 +1,9 @@
+import six
 from ckanext.files.shared import make_collector
 
+if six.PY3:
+    from typing import Any
+
 _auth_functions, auth = make_collector()
 
 
@@ -9,24 +13,29 @@ def get_auth_functions():
 
 @auth
 def files_file_create(context, data_dict):
+    # type: (Any, dict[str, Any]) -> Any
     return {"success": False}
 
 
 @auth
 def files_file_update(context, data_dict):
+    # type: (Any, dict[str, Any]) -> Any
     return {"success": False}
 
 
 @auth
 def files_file_delete(context, data_dict):
+    # type: (Any, dict[str, Any]) -> Any
     return {"success": False}
 
 
 @auth
 def files_file_show(context, data_dict):
+    # type: (Any, dict[str, Any]) -> Any
     return {"success": True}
 
 
 @auth
 def files_get_unused_files(context, data_dict):
+    # type: (Any, dict[str, Any]) -> Any
     return {"success": False}

ckanext/files/model/file.py

@@ -1,34 +1,32 @@
-from __future__ import annotations
+import six
 
 import datetime
 
 
 from sqlalchemy import Column, UnicodeText, DateTime
-from sqlalchemy.orm import Query
 from sqlalchemy.dialects.postgresql import JSONB
 
 import ckan.plugins.toolkit as tk
-import ckan.model as model
 from ckan.model.types import make_uuid
 from ckan.lib.dictization import table_dictize
 from .base import Base
 
+if six.PY3:
+    from typing import Any
 
-class File(Base):
+
+class File(Base):  # type: ignore
     __tablename__ = "files_file"
     id = Column(UnicodeText, primary_key=True, default=make_uuid)
     name = Column(UnicodeText, nullable=False)
     path = Column(UnicodeText, nullable=False)
     kind = Column(UnicodeText, nullable=False)
-    uploaded_at = Column(
-        DateTime, nullable=False, default=datetime.datetime.utcnow
-    )
-    last_access = Column(
-        DateTime, nullable=False, default=datetime.datetime.utcnow
-    )
+    uploaded_at = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
+    last_access = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     extras = Column(JSONB)
 
     def dictize(self, context):
+        # type: (Any) -> dict[str, Any]
         result = table_dictize(self, context)
         result["url"] = tk.h.url_for("files.get_file", file_id=self.id, qualified=True)
         return result