Merge remote-tracking branch 'origin/master' into file-manager

README.md

@@ -88,6 +88,16 @@ the file, which config options will be applied, etc. Default value is
 
   Returns: `True`
 
+
+* `files_get_unused_files`
+
+  List of all `file`s, that are not used for N days.
+
+  Parameters:
+  * `threshold: Optional[int]`: threshold in days, default `180`.
+
+  Returns: `True`
+
 ## Config settings
 
 	# Allowed size for uploaded file in MB.

ckanext/files/cli.py

@@ -0,0 +1,53 @@
+import click
+
+from ckan.plugins import toolkit as tk
+
+import ckanext.files.config as files_conf
+
+__all__ = [
+    "files",
+]
+
+
+@click.group(short_help="ckanext-files CLI commands")
+def files():
+    pass
+
+
+@files.command()
+@click.option("--delete", "-d", is_flag=True, help="Delete orphaned datasets.")
+@click.argument("threshold", required=False, type=int)
+def remove_unused_files(delete: bool, threshold: int):
+    """Remove files that are not used for N days. The unused threshold is specified
+    in a config"""
+    threshold = (
+        threshold
+        if threshold is not None
+        else files_conf.get_unused_threshold()
+    )
+
+    files = tk.get_action("files_get_unused_files")(
+        {"ignore_auth": True}, {"threshold": threshold}
+    )
+
+    if not files:
+        return click.secho("No unused files", fg="blue")
+
+    click.secho(
+        f"Found unused files that were unused more than {threshold} days:", fg="green"
+    )
+
+    for file in files:
+        click.echo(f"File path={file['path']}")
+
+        if delete:
+            tk.get_action("files_file_delete")(
+                {"ignore_auth": True}, {"id": file["id"]}
+            )
+            click.echo(f"File was deleted", fg="red")
+
+    if not delete:
+        click.secho(
+            "If you want to delete unused files, add `--delete` flag",
+            fg="red",
+        )

ckanext/files/config.py

@@ -0,0 +1,7 @@
+import ckan.plugins.toolkit as tk
+
+CONF_UNUSED_THRESHOLD = "ckanext.files.unused_threshold"
+
+
+def get_unused_threshold() -> int:
+    return tk.config[CONF_UNUSED_THRESHOLD]

ckanext/files/config_declaration.yaml

@@ -0,0 +1,13 @@
+version: 1
+groups:
+  - annotation: ckanext-admin_panel
+    options:
+      - key: ckanext.files.<KIND>.max_size
+        type: dynamic
+        description: File max size in MB for a specific file type
+        default: 2
+
+      - key: ckanext.files.unused_threshold
+        type: int
+        description: Сonsider the file unused if there was no request for access within N days
+        default: 180

ckanext/files/logic/action.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
+import datetime
 import os
 from typing import Any, Optional
+
 import ckan.plugins.toolkit as tk
 from ckan.logic import validate
 from ckan.lib.uploader import get_uploader, get_storage_path
@@ -99,13 +101,16 @@ def file_show(context, data_dict):
     if not file:
         raise tk.ObjectNotFound("File not found")
 
+    file.last_access = datetime.datetime.utcnow()
+    context["session"].commit()
+
     return file.dictize(context)
 
 
 def _remove_file_from_filesystem(file_path: str) -> bool:
     """Remove a file from the file system"""
     storage_path = get_storage_path()
-    file_path = os.path.join(storage_path, 'storage', file_path)
+    file_path = os.path.join(storage_path, "storage", file_path)
 
     if not os.path.exists(file_path):
         # TODO: What are we going to do then? Probably, skip silently
@@ -119,3 +124,24 @@ def _remove_file_from_filesystem(file_path: str) -> bool:
         return False
 
     return True
+
+
+@action
+@tk.side_effect_free
+@validate(schema.file_get_unused_files)
+def get_unused_files(context, data_dict):
+    """Return a list of unused file based on a configured threshold"""
+    tk.check_access("files_get_unused_files", context, data_dict)
+
+    threshold = datetime.datetime.utcnow() - datetime.timedelta(
+        days=data_dict["threshold"]
+    )
+
+    files: list[File] = (
+        context["session"]
+        .query(File)
+        .filter(File.last_access < threshold)
+        .order_by(File.last_access.desc())
+    ).all()
+
+    return [file.dictize(context) for file in files]

ckanext/files/logic/auth.py

@@ -21,3 +21,8 @@ def file_delete(context, data_dict):
 @auth
 def file_show(context, data_dict):
     {"success": False}
+
+
+@auth
+def get_unused_files(context, data_dict):
+    {"success": False}

ckanext/files/logic/schema.py

@@ -35,3 +35,13 @@ def file_update(not_empty, ignore_missing, unicode_safe, dict_only, ignore):
         "extras": [ignore_missing, dict_only],
         "__extras": [ignore],
     }
+
+
+@validator_args
+def file_get_unused_files(int_validator, default):
+    return {
+        "threshold": [
+            default(tk.config["ckanext.files.unused_threshold"]),
+            int_validator,
+        ],
+    }

ckanext/files/migration/files/versions/2c5f1f90888c_add_file_last_access_field.py

@@ -0,0 +1,32 @@
+"""Add file last_access field
+
+Revision ID: 2c5f1f90888c
+Revises: cc1a832108c5
+Create Date: 2024-01-27 12:47:35.568291
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import func
+
+# revision identifiers, used by Alembic.
+revision = "2c5f1f90888c"
+down_revision = "cc1a832108c5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "files_file",
+        sa.Column(
+            "last_access",
+            sa.DateTime(),
+            nullable=False,
+            server_default=func.now(),
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("files_file", "last_access")

ckanext/files/model/file.py

@@ -4,9 +4,11 @@
 
 
 from sqlalchemy import Column, UnicodeText, DateTime
+from sqlalchemy.orm import Query
 from sqlalchemy.dialects.postgresql import JSONB
 
 import ckan.plugins.toolkit as tk
+import ckan.model as model
 from ckan.model.types import make_uuid
 from ckan.lib.dictization import table_dictize
 from .base import Base
@@ -21,11 +23,14 @@ class File(Base):
     uploaded_at = Column(
         DateTime, nullable=False, default=datetime.datetime.utcnow
     )
+    last_access = Column(
+        DateTime, nullable=False, default=datetime.datetime.utcnow
+    )
     extras = Column(JSONB)
 
     def dictize(self, context):
         result = table_dictize(self, context)
-        result["url"] = tk.h.url_for_static(result["path"], qualified=True)
+        result["url"] = tk.h.url_for("files.get_file", file_id=self.id)
         return result
 
     @property

ckanext/files/plugin.py

@@ -4,6 +4,9 @@
 from .logic import action, auth, schema
 
 
+@tk.blanket.blueprints
+@tk.blanket.config_declarations
+@tk.blanket.cli
 class FilesPlugin(p.SingletonPlugin):
     p.implements(p.IActions)
     p.implements(p.IAuthFunctions)

ckanext/files/tests/logic/test_action.py

@@ -1,6 +1,8 @@
 import pytest
+
 import ckan.model as model
 from ckan.tests.helpers import call_action
+
 from ckanext.files.model import File
 
 
@@ -26,7 +28,7 @@ def test_basic_file(self, create_with_upload):
         )
 
         assert result["name"] == "test file"
-        assert result["url"].endswith(filename)
+        assert result["url"] == f"/files/get_url/{result['id']}"
 
 
 @pytest.mark.usefixtures("with_plugins")
@@ -49,10 +51,24 @@ def test_basic_delete(self, random_file):
 
 
 @pytest.mark.usefixtures("with_plugins")
-class TestFileDelete:
+class TestFileShow:
     def test_basic_show(self, random_file):
         result = call_action("files_file_show", id=random_file["id"])
-        assert result == random_file
+        assert result["id"] == random_file["id"]
 
         result = call_action("files_file_show", id=random_file["name"])
-        assert result == random_file
+        assert result["id"] == random_file["id"]
+
+    def test_show_updates_last_access(self, random_file):
+        result = call_action("files_file_show", id=random_file["id"])
+        assert result["last_access"] != random_file["last_access"]
+
+
+@pytest.mark.usefixtures("with_plugins")
+class TestGetUnusedFiles:
+    def test_no_unused_files(self, random_file):
+        assert not call_action("files_get_unused_files")
+
+    @pytest.mark.ckan_config("ckanext.files.unused_threshold", 0)
+    def test_configure_default_threshold(self, random_file, ckan_config):
+        assert call_action("files_get_unused_files")

ckanext/files/views.py

@@ -0,0 +1,40 @@
+import logging
+
+from flask import Blueprint
+from flask.views import MethodView
+
+import ckan.plugins.toolkit as tk
+
+
+log = logging.getLogger(__name__)
+files = Blueprint("files", __name__)
+
+
+class FilesGetFileView(MethodView):
+    """This view is designed for serving files while also updating
+    the 'last_access' field in the database for the corresponding file object.
+
+    The `last_access` field is updated inside `files_file_show` action.
+    """
+
+    def get(self, file_id: str):
+        try:
+            file_data = tk.get_action("files_file_show")(
+                {
+                    "user": tk.current_user.name,
+                    "auth_user_obj": tk.current_user,
+                },
+                {"id": file_id},
+            )
+        except (tk.ValidationError, OSError):
+            return
+
+        return tk.redirect_to(
+            tk.h.url_for_static(file_data["path"], qualified=True)
+        )
+
+
+files.add_url_rule(
+    "/files/get_url/<file_id>",
+    view_func=FilesGetFileView.as_view("get_file"),
+)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = ckanext-files
-version = 0.0.3
+version = 0.1.0
 description =
 long_description = file: README.md
 long_description_content_type = text/markdown