From a937c9be1d17dde59042c5c39bad530dfdf0e7b5 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Thu, 24 Aug 2023 10:02:49 +0200 Subject: [PATCH] [ADD] fs_storage: Get access to filesyste storages This addon define a new model 'fs.storage' used to get access to a filesystem storage (ftp, sftp, s3, azure, ...) through an unified interface provided by the 'fsspec' python library (https://filesystem-spec.readthedocs.io/en/latest) --- fs_storage/README.rst | 35 ++ fs_storage/__init__.py | 6 + fs_storage/__manifest__.py | 22 + fs_storage/demo/fs_storage.xml | 8 + fs_storage/i18n/storage_backend.pot | 144 ++++++ fs_storage/models/__init__.py | 1 + fs_storage/models/fs_storage.py | 409 ++++++++++++++++ fs_storage/odoo_file_system.py | 50 ++ fs_storage/readme/CONTRIBUTORS.rst | 2 + fs_storage/readme/DESCRIPTION.rst | 63 +++ fs_storage/readme/HISTORY.rst | 0 fs_storage/readme/ROADMAP.rst | 9 + fs_storage/readme/USAGE.rst | 84 ++++ fs_storage/rooted_dir_file_system.py | 37 ++ fs_storage/security/ir.model.access.csv | 2 + fs_storage/static/description/icon.png | Bin 0 -> 9455 bytes fs_storage/static/description/index.html | 563 +++++++++++++++++++++++ fs_storage/tests/__init__.py | 1 + fs_storage/tests/test_fs_storage.py | 153 ++++++ fs_storage/views/fs_storage_view.xml | 117 +++++ requirements.txt | 2 + setup/fs_storage/odoo/addons/fs_storage | 1 + setup/fs_storage/setup.py | 6 + 23 files changed, 1715 insertions(+) create mode 100644 fs_storage/README.rst create mode 100644 fs_storage/__init__.py create mode 100644 fs_storage/__manifest__.py create mode 100644 fs_storage/demo/fs_storage.xml create mode 100644 fs_storage/i18n/storage_backend.pot create mode 100644 fs_storage/models/__init__.py create mode 100644 fs_storage/models/fs_storage.py create mode 100644 fs_storage/odoo_file_system.py create mode 100644 fs_storage/readme/CONTRIBUTORS.rst create mode 100644 fs_storage/readme/DESCRIPTION.rst create mode 100644 fs_storage/readme/HISTORY.rst create mode 100644 fs_storage/readme/ROADMAP.rst create mode 100644 fs_storage/readme/USAGE.rst create mode 100644 fs_storage/rooted_dir_file_system.py create mode 100644 fs_storage/security/ir.model.access.csv create mode 100644 fs_storage/static/description/icon.png create mode 100644 fs_storage/static/description/index.html create mode 100644 fs_storage/tests/__init__.py create mode 100644 fs_storage/tests/test_fs_storage.py create mode 100644 fs_storage/views/fs_storage_view.xml create mode 100644 requirements.txt create mode 120000 setup/fs_storage/odoo/addons/fs_storage create mode 100644 setup/fs_storage/setup.py diff --git a/fs_storage/README.rst b/fs_storage/README.rst new file mode 100644 index 0000000000..38929e8775 --- /dev/null +++ b/fs_storage/README.rst @@ -0,0 +1,35 @@ +**This file is going to be generated by oca-gen-addon-readme.** + +*Manual changes will be overwritten.* + +Please provide content in the ``readme`` directory: + +* **DESCRIPTION.rst** (required) +* INSTALL.rst (optional) +* CONFIGURE.rst (optional) +* **USAGE.rst** (optional, highly recommended) +* DEVELOP.rst (optional) +* ROADMAP.rst (optional) +* HISTORY.rst (optional, recommended) +* **CONTRIBUTORS.rst** (optional, highly recommended) +* CREDITS.rst (optional) + +Content of this README will also be drawn from the addon manifest, +from keys such as name, authors, maintainers, development_status, +and license. + +A good, one sentence summary in the manifest is also highly recommended. + + +Automatic changelog generation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`HISTORY.rst` can be auto generated using `towncrier `_. + +Just put towncrier compatible changelog fragments into `readme/newsfragments` +and the changelog file will be automatically generated and updated when a new fragment is added. + +Please refer to `towncrier` documentation to know more. + +NOTE: the changelog will be automatically generated when using `/ocabot merge $option`. +If you need to run it manually, refer to `OCA/maintainer-tools README `_. diff --git a/fs_storage/__init__.py b/fs_storage/__init__.py new file mode 100644 index 0000000000..1df985fe64 --- /dev/null +++ b/fs_storage/__init__.py @@ -0,0 +1,6 @@ +# register protocols first +from . import odoo_file_system +from . import rooted_dir_file_system + +# then add normal imports +from . import models diff --git a/fs_storage/__manifest__.py b/fs_storage/__manifest__.py new file mode 100644 index 0000000000..c66b69f7e4 --- /dev/null +++ b/fs_storage/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "Filesystem Storage Backend", + "summary": "Implement the concept of Storage with amazon S3, sftp...", + "version": "16.0.1.0.1", + "category": "FS Storage", + "website": "https://github.com/OCA/storage", + "author": " ACSONE SA/NV, Odoo Community Association (OCA)", + "license": "LGPL-3", + "development_status": "Beta", + "installable": True, + "depends": ["base", "base_sparse_field", "server_environment"], + "data": [ + "views/fs_storage_view.xml", + "security/ir.model.access.csv", + ], + "demo": ["demo/fs_storage.xml"], + "external_dependencies": {"python": ["fsspec"]}, +} diff --git a/fs_storage/demo/fs_storage.xml b/fs_storage/demo/fs_storage.xml new file mode 100644 index 0000000000..a1fd01fd70 --- /dev/null +++ b/fs_storage/demo/fs_storage.xml @@ -0,0 +1,8 @@ + + + + Odoo Filesystem Backend + odoofs + odoofs + + diff --git a/fs_storage/i18n/storage_backend.pot b/fs_storage/i18n/storage_backend.pot new file mode 100644 index 0000000000..45503812f9 --- /dev/null +++ b/fs_storage/i18n/storage_backend.pot @@ -0,0 +1,144 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fs_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type_env_default +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path_env_default +msgid " Env Default" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/components/filesystem_adapter.py:0 +#, python-format +msgid "Access to %s is forbidden" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type +msgid "Backend Type" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type_env_is_editable +msgid "Backend Type Env Is Editable" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Connection Test Failed!" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Connection Test Succeeded!" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__create_uid +msgid "Created by" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__create_date +msgid "Created on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path +msgid "Directory Path" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path_env_is_editable +msgid "Directory Path Env Is Editable" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__display_name +msgid "Display Name" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Everything seems properly set up!" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields.selection,name:fs_storage.selection__fs_storage__backend_type__filesystem +msgid "Filesystem" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__has_validation +msgid "Has Validation" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__id +msgid "ID" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage____last_update +msgid "Last Modified on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__write_uid +msgid "Last Updated by" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__write_date +msgid "Last Updated on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__name +msgid "Name" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,help:fs_storage.field_fs_storage__directory_path +#: model:ir.model.fields,help:fs_storage.field_fs_storage__directory_path_env_default +msgid "Relative path to the directory to store the file" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__server_env_defaults +msgid "Server Env Defaults" +msgstr "" + +#. module: fs_storage +#: model:ir.actions.act_window,name:fs_storage.act_open_fs_storage_view +#: model:ir.model,name:fs_storage.model_fs_storage +#: model:ir.ui.menu,name:fs_storage.menu_storage +#: model:ir.ui.menu,name:fs_storage.menu_fs_storage +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_form +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_search +msgid "FS Storage" +msgstr "" + +#. module: fs_storage +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_form +msgid "Test connection" +msgstr "" diff --git a/fs_storage/models/__init__.py b/fs_storage/models/__init__.py new file mode 100644 index 0000000000..349bb0495a --- /dev/null +++ b/fs_storage/models/__init__.py @@ -0,0 +1 @@ +from . import fs_storage diff --git a/fs_storage/models/fs_storage.py b/fs_storage/models/fs_storage.py new file mode 100644 index 0000000000..5bcc5f3e5e --- /dev/null +++ b/fs_storage/models/fs_storage.py @@ -0,0 +1,409 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from __future__ import annotations + +import base64 +import functools +import inspect +import json +import logging +import os.path +import re +import warnings +from typing import AnyStr + +import fsspec + +from odoo import _, api, fields, models, tools +from odoo.exceptions import ValidationError + +from odoo.addons.base_sparse_field.models.fields import Serialized + +_logger = logging.getLogger(__name__) + + +# TODO: useful for the whole OCA? +def deprecated(reason): + """Mark functions or classes as deprecated. + + Emit warning at execution. + + The @deprecated is used with a 'reason'. + + .. code-block:: python + + @deprecated("please, use another function") + def old_function(x, y): + pass + """ + + def decorator(func1): + + if inspect.isclass(func1): + fmt1 = "Call to deprecated class {name} ({reason})." + else: + fmt1 = "Call to deprecated function {name} ({reason})." + + @functools.wraps(func1) + def new_func1(*args, **kwargs): + warnings.simplefilter("always", DeprecationWarning) + warnings.warn( + fmt1.format(name=func1.__name__, reason=reason), + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) + return func1(*args, **kwargs) + + return new_func1 + + return decorator + + +class FSStorage(models.Model): + _name = "fs.storage" + _inherit = "server.env.mixin" + _description = "FS Storage" + + __slots__ = ("__fs", "__odoo_storage_path") + + def __init__(self, env, ids=(), prefetch_ids=()): + super().__init__(env, ids=ids, prefetch_ids=prefetch_ids) + self.__fs = None + self.__odoo_storage_path = None + + name = fields.Char(required=True) + code = fields.Char( + required=True, + help="Technical code used to identify the storage backend into the code." + "This code must be unique. This code is used for example to define the " + "storage backend to store the attachments via the configuration parameter " + "'ir_attachment.storage.force.database' when the module 'fs_attachment' " + "is installed.", + ) + protocol = fields.Selection( + selection="_get_protocols", + required=True, + default="odoofs", + help="The protocol used to access the content of filesystem.\n" + "This list is the one supported by the fsspec library (see " + "https://filesystem-spec.readthedocs.io/en/latest). A filesystem protocol" + "is added by default and refers to the odoo local filesystem.\n" + "Pay attention that according to the protocol, some options must be" + "provided through the options field.", + ) + protocol_descr = fields.Text( + compute="_compute_protocol_descr", + ) + options = fields.Text( + help="The options used to initialize the filesystem.\n" + "This is a JSON field that depends on the protocol used.\n" + "For example, for the sftp protocol, you can provide the following:\n" + "{\n" + " 'host': 'my.sftp.server',\n" + " 'ssh_kwrags': {\n" + " 'username': 'myuser',\n" + " 'password': 'mypassword',\n" + " 'port': 22,\n" + " }\n" + "}\n" + "For more information, please refer to the fsspec documentation:\n" + "https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations" + ) + + json_options = Serialized( + help="The options used to initialize the filesystem.\n", + compute="_compute_json_options", + inverse="_inverse_json_options", + ) + directory_path = fields.Char( + help="Relative path to the directory to store the file" + ) + + # the next fields are used to display documentation to help the user + # to configure the backend + options_protocol = fields.Selection( + string="Describes Protocol", + selection="_get_options_protocol", + default="odoofs", + help="The protocol used to access the content of filesystem.\n" + "This list is the one supported by the fsspec library (see " + "https://filesystem-spec.readthedocs.io/en/latest). A filesystem protocol" + "is added by default and refers to the odoo local filesystem.\n" + "Pay attention that according to the protocol, some options must be" + "provided through the options field.", + store=False, + ) + options_properties = fields.Text( + string="Available properties", + compute="_compute_options_properties", + store=False, + ) + + _sql_constraints = [ + ( + "code_uniq", + "unique(code)", + "The code must be unique", + ), + ] + + _server_env_section_name_field = "code" + + @property + def _server_env_fields(self): + return {"protocol": {}, "options": {}, "directory_path": {}} + + def write(self, vals): + self.__fs = None + self.clear_caches() + return super().write(vals) + + @api.model + @tools.ormcache() + def get_id_by_code_map(self): + """Return a dictionary with the code as key and the id as value.""" + return {rec.code: rec.id for rec in self.search([])} + + @api.model + def get_id_by_code(self, code): + """Return the id of the filesystem associated to the given code.""" + return self.get_id_by_code_map().get(code) + + @api.model + def get_by_code(self, code) -> FSStorage: + """Return the filesystem associated to the given code.""" + res = self.browse() + res_id = self.get_id_by_code(code) + if res_id: + res = self.browse(res_id) + return res + + @api.model + @tools.ormcache() + def get_storage_codes(self): + """Return the list of codes of the existing filesystems.""" + return [s.code for s in self.search([])] + + @api.model + @tools.ormcache("code") + def get_fs_by_code(self, code): + """Return the filesystem associated to the given code. + + :param code: the code of the filesystem + """ + fs = None + fs_storage = self.get_by_code(code) + if fs_storage: + fs = fs_storage.fs + return fs + + def copy(self, default=None): + default = default or {} + if "code" not in default: + default["code"] = "{}_copy".format(self.code) + return super().copy(default) + + @api.model + def _get_protocols(self) -> list[tuple[str, str]]: + protocol = [("odoofs", "Odoo's FileSystem")] + for p in fsspec.available_protocols(): + try: + cls = fsspec.get_filesystem_class(p) + protocol.append((p, f"{p} ({cls.__name__})")) + except ImportError as e: + _logger.debug("Cannot load the protocol %s. Reason: %s", p, e) + return protocol + + @api.constrains("options") + def _check_options(self) -> None: + for rec in self: + try: + json.loads(rec.options or "{}") + except Exception as e: + raise ValidationError(_("The options must be a valid JSON")) from e + + @api.depends("options") + def _compute_json_options(self) -> None: + for rec in self: + rec.json_options = json.loads(rec.options or "{}") + + def _inverse_json_options(self) -> None: + for rec in self: + rec.options = json.dumps(rec.json_options) + + @api.depends("protocol") + def _compute_protocol_descr(self) -> None: + for rec in self: + rec.protocol_descr = fsspec.get_filesystem_class(rec.protocol).__doc__ + + @api.model + def _get_options_protocol(self) -> list[tuple[str, str]]: + protocol = [("odoofs", "Odoo's Filesystem")] + for p in fsspec.available_protocols(): + try: + fsspec.get_filesystem_class(p) + protocol.append((p, p)) + except ImportError as e: + _logger.debug("Cannot load the protocol %s. Reason: %s", p, e) + return protocol + + @api.depends("options_protocol") + def _compute_options_properties(self) -> None: + for rec in self: + cls = fsspec.get_filesystem_class(rec.options_protocol) + signature = inspect.signature(cls.__init__) + doc = inspect.getdoc(cls.__init__) + rec.options_properties = f"__init__{signature}\n{doc}" + + @property + def fs(self) -> fsspec.AbstractFileSystem: + """Get the fsspec filesystem for this backend.""" + self.ensure_one() + if not self.__fs: + self.__fs = self._get_filesystem() + return self.__fs + + def _get_filesystem_storage_path(self) -> str: + """Get the path to the storage directory. + + This path is relative to the odoo filestore.and is used as root path + when the protocol is filesystem. + """ + self.ensure_one() + path = os.path.join(self.env["ir.attachment"]._filestore(), "storage") + if not os.path.exists(path): + os.makedirs(path) + return path + + @property + def _odoo_storage_path(self) -> str: + """Get the path to the storage directory. + + This path is relative to the odoo filestore.and is used as root path + when the protocol is filesystem. + """ + if not self.__odoo_storage_path: + self.__odoo_storage_path = self._get_filesystem_storage_path() + return self.__odoo_storage_path + + def _recursive_add_odoo_storage_path(self, options: dict) -> dict: + """Add the odoo storage path to the options. + + This is a recursive function that will add the odoo_storage_path + option to the nested target_options if the target_protocol is + odoofs + """ + if "target_protocol" in options: + target_options = options.get("target_options", {}) + if options["target_protocol"] == "odoofs": + target_options["odoo_storage_path"] = self._odoo_storage_path + options["target_options"] = target_options + self._recursive_add_odoo_storage_path(target_options) + return options + + def _get_filesystem(self) -> fsspec.AbstractFileSystem: + """Get the fsspec filesystem for this backend. + + See https://filesystem-spec.readthedocs.io/en/latest/api.html + #fsspec.spec.AbstractFileSystem + + :return: fsspec.AbstractFileSystem + """ + self.ensure_one() + options = self.json_options + if self.protocol == "odoofs": + options["odoo_storage_path"] = self._odoo_storage_path + options = self._recursive_add_odoo_storage_path(options) + fs = fsspec.filesystem(self.protocol, **options) + directory_path = self.directory_path + if directory_path: + fs = fsspec.filesystem("rooted_dir", path=directory_path, fs=fs) + return fs + + # Deprecated methods used to ease the migration from the storage_backend addons + # to the fs_storage addons. These methods will be removed in the future (Odoo 18) + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def add(self, relative_path, data, binary=True, **kwargs) -> None: + if not binary: + data = base64.b64decode(data) + path = relative_path.split(self.fs.sep)[:-1] + if not self.fs.exists(self.fs.sep.join(path)): + self.fs.makedirs(self.fs.sep.join(path)) + with self.fs.open(relative_path, "wb", **kwargs) as f: + f.write(data) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def get(self, relative_path, binary=True, **kwargs) -> AnyStr: + data = self.fs.read_bytes(relative_path, **kwargs) + if not binary and data: + data = base64.b64encode(data) + return data + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def list_files(self, relative_path="", pattern=False) -> list[str]: + relative_path = relative_path or self.fs.root_marker + if not self.fs.exists(relative_path): + return [] + if pattern: + relative_path = self.fs.sep.join([relative_path, pattern]) + return self.fs.glob(relative_path) + return self.fs.ls(relative_path, detail=False) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def find_files(self, pattern, relative_path="", **kw) -> list[str]: + """Find files matching given pattern. + + :param pattern: regex expression + :param relative_path: optional relative path containing files + :return: list of file paths as full paths from the root + """ + result = [] + relative_path = relative_path or self.fs.root_marker + if not self.fs.exists(relative_path): + return [] + regex = re.compile(pattern) + for file_path in self.fs.ls(relative_path, detail=False): + if regex.match(file_path): + result.append(file_path) + return result + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def move_files(self, files, destination_path, **kw) -> None: + """Move files to given destination. + + :param files: list of file paths to be moved + :param destination_path: directory path where to move files + :return: None + """ + for file_path in files: + self.fs.move( + file_path, + self.fs.sep.join([destination_path, os.path.basename(file_path)]), + **kw, + ) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def delete(self, relative_path) -> None: + self.fs.rm_file(relative_path) + + def action_test_config(self) -> None: + try: + self.fs.ls("", detail=False) + title = _("Connection Test Succeeded!") + message = _("Everything seems properly set up!") + msg_type = "success" + except Exception as err: + title = _("Connection Test Failed!") + message = str(err) + msg_type = "danger" + return { + "type": "ir.actions.client", + "tag": "display_notification", + "params": { + "title": title, + "message": message, + "type": msg_type, + "sticky": False, + }, + } diff --git a/fs_storage/odoo_file_system.py b/fs_storage/odoo_file_system.py new file mode 100644 index 0000000000..1827a31acc --- /dev/null +++ b/fs_storage/odoo_file_system.py @@ -0,0 +1,50 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + + +from fsspec.registry import register_implementation + +from .rooted_dir_file_system import RootedDirFileSystem + + +class OdooFileSystem(RootedDirFileSystem): + """A directory-based filesystem for Odoo. + + This filesystem is mounted from a specific subdirectory of the Odoo + filestore directory. + + It extends the RootedDirFileSystem to avoid going outside the + specific subdirectory nor the Odoo filestore directory. + + Parameters: + odoo_storage_path: The path of the subdirectory of the Odoo filestore + directory to mount. This parameter is required and is always provided + by the Odoo FS Storage even if it is explicitly defined in the + storage options. + fs: AbstractFileSystem + An instantiated filesystem to wrap. + target_protocol, target_options: + if fs is none, construct it from these + """ + + def __init__( + self, + *, + odoo_storage_path, + fs=None, + target_protocol=None, + target_options=None, + **storage_options + ): + if not odoo_storage_path: + raise ValueError("odoo_storage_path is required") + super().__init__( + path=odoo_storage_path, + fs=fs, + target_protocol=target_protocol, + target_options=target_options, + **storage_options + ) + + +register_implementation("odoofs", OdooFileSystem) diff --git a/fs_storage/readme/CONTRIBUTORS.rst b/fs_storage/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..60c32f1d3f --- /dev/null +++ b/fs_storage/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Laurent Mignon +* Sébastien BEAU diff --git a/fs_storage/readme/DESCRIPTION.rst b/fs_storage/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..1cfc5c1a00 --- /dev/null +++ b/fs_storage/readme/DESCRIPTION.rst @@ -0,0 +1,63 @@ +This addon is a technical addon that allows you to define filesystem like +storage for your data. It's used by other addons to store their data in a +transparent way into different kind of storages. + +Through the fs.storage record, you get access to an object that implements +the `fsspec.spec.AbstractFileSystem `_ interface and therefore give +you an unified interface to access your data whatever the storage protocol you +decide to use. + +The list of supported protocols depends on the installed fsspec implementations. +By default, the addon will install the following protocols: + +* LocalFileSystem +* MemoryFileSystem +* ZipFileSystem +* TarFileSystem +* FTPFileSystem +* CachingFileSystem +* WholeFileSystem +* SimplCacheFileSystem +* ReferenceFileSystem +* GenericFileSystem +* DirFileSystem +* DatabricksFileSystem +* GitHubFileSystem +* JupiterFileSystem +* OdooFileSystem + +The OdooFileSystem is the one that allows you to store your data into a directory +mounted into your Odoo's storage directory. This is the default FS Storage +when creating a new fs.storage record. + +Others protocols are available through the installation of additional +python packages: + +* DropboxDriveFileSystem -> `pip install fsspec[dropbox]` +* HTTPFileSystem -> `pip install fsspec[http]` +* HTTPSFileSystem -> `pip install fsspec[http]` +* GCSFileSystem -> `pip install fsspec[gcs]` +* GSFileSystem -> `pip install fsspec[gs]` +* GoogleDriveFileSystem -> `pip install gdrivefs` +* SFTPFileSystem -> `pip install fsspec[sftp]` +* HaddoopFileSystem -> `pip install fsspec[hdfs]` +* S3FileSystem -> `pip install fsspec[s3]` +* WandbFS -> `pip install wandbfs` +* OCIFileSystem -> `pip install fsspec[oci]` +* AsyncLocalFileSystem -> `pip install 'morefs[asynclocalfs]` +* AzureDatalakeFileSystem -> `pip install fsspec[adl]` +* AzureBlobFileSystem -> `pip install fsspec[abfs]` +* DaskWorkerFileSystem -> `pip install fsspec[dask]` +* GitFileSystem -> `pip install fsspec[git]` +* SMBFileSystem -> `pip install fsspec[smb]` +* LibArchiveFileSystem -> `pip install fsspec[libarchive]` +* OSSFileSystem -> `pip install ossfs` +* WebdavFileSystem -> `pip install webdav4` +* DVCFileSystem -> `pip install dvc` +* XRootDFileSystem -> `pip install fsspec-xrootd` + +This list of supported protocols is not exhaustive or could change in the future +depending on the fsspec releases. You can find more information about the +supported protocols on the `fsspec documentation +`_. diff --git a/fs_storage/readme/HISTORY.rst b/fs_storage/readme/HISTORY.rst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fs_storage/readme/ROADMAP.rst b/fs_storage/readme/ROADMAP.rst new file mode 100644 index 0000000000..0c697994d0 --- /dev/null +++ b/fs_storage/readme/ROADMAP.rst @@ -0,0 +1,9 @@ +* Transactions: fsspec comes with a transactional mechanism that once started, + gathers all the files created during the transaction, and if the transaction + is committed, moves them to their final locations. It would be useful to + bridge this with the transactional mechanism of odoo. This would allow to + ensure that all the files created during a transaction are either all + moved to their final locations, or all deleted if the transaction is rolled + back. This mechanism is only valid for files created during the transaction + by a call to the `open` method of the file system. It is not valid for others + operations, such as `rm`, `mv_file`, ... . diff --git a/fs_storage/readme/USAGE.rst b/fs_storage/readme/USAGE.rst new file mode 100644 index 0000000000..f65fb39af2 --- /dev/null +++ b/fs_storage/readme/USAGE.rst @@ -0,0 +1,84 @@ +Configuration +~~~~~~~~~~~~~ + +When you create a new backend, you must specify the following: + +* The name of the backend. This is the name that will be used to + identify the backend into Odoo +* The code of the backend. This code will identify the backend into the store_fname + field of the ir.attachment model. This code must be unique. It will be used + as scheme. example of the store_fname field: ``odoofs://abs34Tg11``. +* The protocol used by the backend. The protocol refers to the supported + protocols of the fsspec python package. +* A directory path. This is a root directory from which the filesystem will + be mounted. This directory must exist. +* The protocol options. These are the options that will be passed to the + fsspec python package when creating the filesystem. These options depend + on the protocol used and are described in the fsspec documentation. + +Some protocols defined in the fsspec package are wrappers around other +protocols. For example, the SimpleCacheFileSystem protocol is a wrapper +around any local filesystem protocol. In such cases, you must specify into the +protocol options the protocol to be wrapped and the options to be passed to +the wrapped protocol. + +For example, if you want to create a backend that uses the SimpleCacheFileSystem +protocol, after selecting the SimpleCacheFileSystem protocol, you must specify +the protocol options as follows: + +.. code-block:: python + + { + "directory_path": "/tmp/my_backend", + "target_protocol": "odoofs", + "target_options": {...}, + } + +In this example, the SimpleCacheFileSystem protocol will be used as a wrapper +around the odoofs protocol. + +Server Environment +~~~~~~~~~~~~~~~~~~ + +To ease the management of the filesystem storages configuration accross the different +environments, the configuration of the filesystem storages can be defined in +environment files or directly in the main configuration file. For example, the +configuration of a filesystem storage with the code `fsprod` can be provided in the +main configuration file as follows: + +.. code-block:: ini + + [fs_storage.fsprod] + protocol=s3 + options={"endpoint_url": "https://my_s3_server/", "key": "KEY", "secret": "SECRET"} + directory_path=my_bucket + +To work, a `storage.backend` record must exist with the code `fsprod` into the database. +In your configuration section, you can specify the value for the following fields: + +* `protocol` +* `options` +* `directory_path` + +Migration from storage_backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The fs_storage addon can be used to replace the storage_backend addon. (It has +been designed to be a drop-in replacement for the storage_backend addon). To +ease the migration, the `fs.storage` model defines the high-level methods +available in the storage_backend model. These methods are: + +* `add` +* `get` +* `list_files` +* `find_files` +* `move_files` +* `delete` + +These methods are wrappers around the methods of the `fsspec.AbstractFileSystem` +class (see https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem). +These methods are marked as deprecated and will be removed in a future version (V18) +of the addon. You should use the methods of the `fsspec.AbstractFileSystem` class +instead since they are more flexible and powerful. You can access the instance +of the `fsspec.AbstractFileSystem` class using the `fs` property of a `fs.storage` +record. diff --git a/fs_storage/rooted_dir_file_system.py b/fs_storage/rooted_dir_file_system.py new file mode 100644 index 0000000000..8fb401685f --- /dev/null +++ b/fs_storage/rooted_dir_file_system.py @@ -0,0 +1,37 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import os + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.local import make_path_posix +from fsspec.registry import register_implementation + + +class RootedDirFileSystem(DirFileSystem): + """A directory-based filesystem that uses path as a root. + + The main purpose of this filesystem is to ensure that paths are always + a sub path of the initial path. IOW, it is not possible to go outside + the initial path. That's the only difference with the DirFileSystem provided + by fsspec. + + This one should be provided by fsspec itself. We should propose a PR. + """ + + def _join(self, path): + path = super()._join(path) + # Ensure that the path is a subpath of the root path by resolving + # any relative paths. + # Since the path separator is not always the same on all systems, + # we need to normalize the path separator. + path_posix = os.path.normpath(make_path_posix(path, self.sep)) + root_posix = os.path.normpath(make_path_posix(self.path)) + if not path_posix.startswith(root_posix): + raise PermissionError( + "Path %s is not a subpath of the root path %s" % (path, self.path) + ) + return path + + +register_implementation("rooted_dir", RootedDirFileSystem) diff --git a/fs_storage/security/ir.model.access.csv b/fs_storage/security/ir.model.access.csv new file mode 100644 index 0000000000..9c079b2eef --- /dev/null +++ b/fs_storage/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_fs_storage_edit,fs_storage edit,model_fs_storage,base.group_system,1,1,1,1 diff --git a/fs_storage/static/description/icon.png b/fs_storage/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/fs_storage/static/description/index.html b/fs_storage/static/description/index.html new file mode 100644 index 0000000000..e1730e320d --- /dev/null +++ b/fs_storage/static/description/index.html @@ -0,0 +1,563 @@ + + + + + + +Filesystem Storage Backend + + + +
+

Filesystem Storage Backend

+ + +

Beta License: LGPL-3 OCA/storage Translate me on Weblate Try me on Runbot

+

This addon is a technical addon that allows you to define filesystem like +storage for your data. It’s used by other addons to store their data in a +transparent way into different kind of storages.

+

Through the fs.storage record, you get access to an object that implements +the fsspec.spec.AbstractFileSystem interface and therefore give +you an unified interface to access your data whatever the storage protocol you +decide to use.

+

The list of supported protocols depends on the installed fsspec implementations. +By default, the addon will install the following protocols:

+
    +
  • LocalFileSystem
    • +
  • MemoryFileSystem
    • +
  • ZipFileSystem
    • +
  • TarFileSystem
    • +
  • FTPFileSystem
    • +
  • CachingFileSystem
    • +
  • WholeFileSystem
    • +
  • SimplCacheFileSystem
    • +
  • ReferenceFileSystem
    • +
  • GenericFileSystem
    • +
  • DirFileSystem
    • +
  • DatabricksFileSystem
    • +
  • GitHubFileSystem
    • +
  • JupiterFileSystem
    • +
  • OdooFileSystem
    • +
+

The OdooFileSystem is the one that allows you to store your data into a directory +mounted into your Odoo’s storage directory. This is the default FS Storage +when creating a new fs.storage record.

+

Others protocols are available through the installation of additional +python packages:

+
    +
  • DropboxDriveFileSystem -> pip install fsspec[dropbox]
    • +
  • HTTPFileSystem -> pip install fsspec[http]
    • +
  • HTTPSFileSystem -> pip install fsspec[http]
    • +
  • GCSFileSystem -> pip install fsspec[gcs]
    • +
  • GSFileSystem -> pip install fsspec[gs]
    • +
  • GoogleDriveFileSystem -> pip install gdrivefs
    • +
  • SFTPFileSystem -> pip install fsspec[sftp]
    • +
  • HaddoopFileSystem -> pip install fsspec[hdfs]
    • +
  • S3FileSystem -> pip install fsspec[s3]
    • +
  • WandbFS -> pip install wandbfs
    • +
  • OCIFileSystem -> pip install fsspec[oci]
    • +
  • AsyncLocalFileSystem -> pip install ‘morefs[asynclocalfs]
    • +
  • AzureDatalakeFileSystem -> pip install fsspec[adl]
    • +
  • AzureBlobFileSystem -> pip install fsspec[abfs]
    • +
  • DaskWorkerFileSystem -> pip install fsspec[dask]
    • +
  • GitFileSystem -> pip install fsspec[git]
    • +
  • SMBFileSystem -> pip install fsspec[smb]
    • +
  • LibArchiveFileSystem -> pip install fsspec[libarchive]
    • +
  • OSSFileSystem -> pip install ossfs
    • +
  • WebdavFileSystem -> pip install webdav4
    • +
  • DVCFileSystem -> pip install dvc
    • +
  • XRootDFileSystem -> pip install fsspec-xrootd
    • +
+

This list of supported protocols is not exhaustive or could change in the future +depending on the fsspec releases. You can find more information about the +supported protocols on the fsspec documentation.

+

Table of contents

+ +
+

Usage

+
+

Configuration

+

When you create a new backend, you must specify the following:

+
    +
  • The name of the backend. This is the name that will be used to +identify the backend into Odoo
    • +
  • The code of the backend. This code will identify the backend into the store_fname +field of the ir.attachment model. This code must be unique. It will be used +as scheme. example of the store_fname field: odoofs://abs34Tg11.
    • +
  • The protocol used by the backend. The protocol refers to the supported +protocols of the fsspec python package.
    • +
  • A directory path. This is a root directory from which the filesystem will +be mounted. This directory must exist.
    • +
  • The protocol options. These are the options that will be passed to the +fsspec python package when creating the filesystem. These options depend +on the protocol used and are described in the fsspec documentation.
    • +
+

Some protocols defined in the fsspec package are wrappers around other +protocols. For example, the SimpleCacheFileSystem protocol is a wrapper +around any local filesystem protocol. In such cases, you must specify into the +protocol options the protocol to be wrapped and the options to be passed to +the wrapped protocol.

+

For example, if you want to create a backend that uses the SimpleCacheFileSystem +protocol, after selecting the SimpleCacheFileSystem protocol, you must specify +the protocol options as follows:

+
+{
+    "directory_path": "/tmp/my_backend",
+    "target_protocol": "odoofs",
+    "target_options": {...},
+}
+
+

In this example, the SimpleCacheFileSystem protocol will be used as a wrapper +around the odoofs protocol.

+
+
+

Server Environment

+

To ease the management of the filesystem storages configuration accross the different +environments, the configuration of the filesystem storages can be defined in +environment files or directly in the main configuration file. For example, the +configuration of a filesystem storage with the code fsprod can be provided in the +main configuration file as follows:

+
+[fs_storage.fsprod]
+protocol=s3
+options={"endpoint_url": "https://my_s3_server/", "key": "KEY", "secret": "SECRET"}
+directory_path=my_bucket
+
+

To work, a storage.backend record must exist with the code fsprod into the database. +In your configuration section, you can specify the value for the following fields:

+
    +
  • protocol
    • +
  • options
    • +
  • directory_path
    • +
+
+
+
+

Known issues / Roadmap

+
    +
  • Transactions: fsspec comes with a transactional mechanism that once started, +gathers all the files created during the transaction, and if the transaction +is committed, moves them to their final locations. It would be useful to +bridge this with the transactional mechanism of odoo. This would allow to +ensure that all the files created during a transaction are either all +moved to their final locations, or all deleted if the transaction is rolled +back. This mechanism is only valid for files created during the transaction +by a call to the open method of the file system. It is not valid for others +operations, such as rm, mv_file, … .
    • +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/storage project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/fs_storage/tests/__init__.py b/fs_storage/tests/__init__.py new file mode 100644 index 0000000000..84bbda192b --- /dev/null +++ b/fs_storage/tests/__init__.py @@ -0,0 +1 @@ +from . import test_fs_storage diff --git a/fs_storage/tests/test_fs_storage.py b/fs_storage/tests/test_fs_storage.py new file mode 100644 index 0000000000..74dfde8814 --- /dev/null +++ b/fs_storage/tests/test_fs_storage.py @@ -0,0 +1,153 @@ +# Copyright 2023 ACSONE SA/NV (http://acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +import base64 +import shutil +import tempfile +import warnings +from unittest import mock + +from odoo.tests.common import TransactionCase +from odoo.tools import mute_logger + +from ..models.fs_storage import FSStorage + + +class TestFSStorage(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + cls.backend: FSStorage = cls.env.ref("fs_storage.default_fs_storage") + cls.backend.json_options = {"target_options": {"auto_mkdir": "True"}} + cls.filedata = base64.b64encode(b"This is a simple file") + cls.filename = "test_file.txt" + cls.case_with_subdirectory = "subdirectory/here" + cls.demo_user = cls.env.ref("base.user_demo") + + def setUp(self): + super().setUp() + mocked_backend = mock.patch.object( + self.backend.__class__, "_get_filesystem_storage_path" + ) + mocked_get_filesystem_storage_path = mocked_backend.start() + tempdir = tempfile.mkdtemp() + mocked_get_filesystem_storage_path.return_value = tempdir + + # pylint: disable=unused-variable + @self.addCleanup + def stop_mock(): + mocked_backend.stop() + # recursively delete the tempdir + shutil.rmtree(tempdir) + + def _create_file(self, backend: FSStorage, filename: str, filedata: str): + with backend.fs.open(filename, "wb") as f: + f.write(filedata) + + @mute_logger("py.warnings") + def _test_deprecated_setting_and_getting_data(self): + # Check that the directory is empty + warnings.filterwarnings("ignore") + files = self.backend.list_files() + self.assertNotIn(self.filename, files) + + # Add a new file + self.backend.add( + self.filename, self.filedata, mimetype="text/plain", binary=False + ) + + # Check that the file exist + files = self.backend.list_files() + self.assertIn(self.filename, files) + + # Retrieve the file added + data = self.backend.get(self.filename, binary=False) + self.assertEqual(data, self.filedata) + + # Delete the file + self.backend.delete(self.filename) + files = self.backend.list_files() + self.assertNotIn(self.filename, files) + + @mute_logger("py.warnings") + def _test_deprecated_find_files(self): + warnings.filterwarnings("ignore") + self.backend.add( + self.filename, self.filedata, mimetype="text/plain", binary=False + ) + try: + res = self.backend.find_files(r".*\.txt") + self.assertListEqual([self.filename], res) + res = self.backend.find_files(r".*\.text") + self.assertListEqual([], res) + finally: + self.backend.delete(self.filename) + + def test_deprecated_setting_and_getting_data_from_root(self): + self._test_deprecated_setting_and_getting_data() + + def test_deprecated_setting_and_getting_data_from_dir(self): + self.backend.directory_path = self.case_with_subdirectory + self._test_deprecated_setting_and_getting_data() + + def test_deprecated_find_files_from_root(self): + self._test_deprecated_find_files() + + def test_deprecated_find_files_from_dir(self): + self.backend.directory_path = self.case_with_subdirectory + self._test_deprecated_find_files() + + def test_ensure_one_fs_by_record(self): + # in this test we ensure that we've one fs by record + backend_ids = [] + for i in range(4): + backend_ids.append( + self.backend.create( + {"name": f"name{i}", "directory_path": f"{i}", "code": f"code{i}"} + ).id + ) + records = self.backend.browse(backend_ids) + fs = None + for rec in records: + self.assertNotEqual(fs, rec.fs) + + def test_relative_access(self): + self.backend.directory_path = self.case_with_subdirectory + self._create_file(self.backend, self.filename, self.filedata) + other_subdirectory = "other_subdirectory" + backend2 = self.backend.copy({"directory_path": other_subdirectory}) + self._create_file(backend2, self.filename, self.filedata) + with self.assertRaises(PermissionError), self.env.cr.savepoint(): + # check that we can't access outside the subdirectory + backend2.fs.ls("../") + with self.assertRaises(PermissionError), self.env.cr.savepoint(): + # check that we can't access the file into another subdirectory + backend2.fs.ls(f"../{self.case_with_subdirectory}") + self.backend.fs.rm_file(self.filename) + backend2.fs.rm_file(self.filename) + + def test_recursive_add_odoo_storage_path_to_options(self): + options = { + "directory_path": "/tmp/my_backend", + "target_protocol": "odoofs", + } + self.backend._recursive_add_odoo_storage_path(options) + self.assertEqual( + self.backend._odoo_storage_path, + options.get("target_options").get("odoo_storage_path"), + ) + options = { + "directory_path": "/tmp/my_backend", + "target_protocol": "dir", + "target_options": { + "path": "/my_backend", + "target_protocol": "odoofs", + }, + } + self.backend._recursive_add_odoo_storage_path(options) + self.assertEqual( + self.backend._odoo_storage_path, + options.get("target_options") + .get("target_options") + .get("odoo_storage_path"), + ) diff --git a/fs_storage/views/fs_storage_view.xml b/fs_storage/views/fs_storage_view.xml new file mode 100644 index 0000000000..3027f29152 --- /dev/null +++ b/fs_storage/views/fs_storage_view.xml @@ -0,0 +1,117 @@ + + + + fs.storage.tree (in fs_storage) + fs.storage + + + + + + + + + + fs.storage.form (in fs_storage) + fs.storage + +
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + fs.storage.search (in fs_storage) + fs.storage + + + + + + + + + FS Storage + ir.actions.act_window + fs.storage + tree,form + + [] + {} + + + + + form + + + + + + tree + + + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..73bcbf6d02 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# generated from manifests external_dependencies +fsspec diff --git a/setup/fs_storage/odoo/addons/fs_storage b/setup/fs_storage/odoo/addons/fs_storage new file mode 120000 index 0000000000..fb821ff7a8 --- /dev/null +++ b/setup/fs_storage/odoo/addons/fs_storage @@ -0,0 +1 @@ +../../../../fs_storage \ No newline at end of file diff --git a/setup/fs_storage/setup.py b/setup/fs_storage/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/fs_storage/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)