diff --git a/fs_storage/README.rst b/fs_storage/README.rst new file mode 100644 index 0000000000..38929e8775 --- /dev/null +++ b/fs_storage/README.rst @@ -0,0 +1,35 @@ +**This file is going to be generated by oca-gen-addon-readme.** + +*Manual changes will be overwritten.* + +Please provide content in the ``readme`` directory: + +* **DESCRIPTION.rst** (required) +* INSTALL.rst (optional) +* CONFIGURE.rst (optional) +* **USAGE.rst** (optional, highly recommended) +* DEVELOP.rst (optional) +* ROADMAP.rst (optional) +* HISTORY.rst (optional, recommended) +* **CONTRIBUTORS.rst** (optional, highly recommended) +* CREDITS.rst (optional) + +Content of this README will also be drawn from the addon manifest, +from keys such as name, authors, maintainers, development_status, +and license. + +A good, one sentence summary in the manifest is also highly recommended. + + +Automatic changelog generation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`HISTORY.rst` can be auto generated using `towncrier `_. + +Just put towncrier compatible changelog fragments into `readme/newsfragments` +and the changelog file will be automatically generated and updated when a new fragment is added. + +Please refer to `towncrier` documentation to know more. + +NOTE: the changelog will be automatically generated when using `/ocabot merge $option`. +If you need to run it manually, refer to `OCA/maintainer-tools README `_. diff --git a/fs_storage/__init__.py b/fs_storage/__init__.py new file mode 100644 index 0000000000..1df985fe64 --- /dev/null +++ b/fs_storage/__init__.py @@ -0,0 +1,6 @@ +# register protocols first +from . import odoo_file_system +from . import rooted_dir_file_system + +# then add normal imports +from . import models diff --git a/fs_storage/__manifest__.py b/fs_storage/__manifest__.py new file mode 100644 index 0000000000..c66b69f7e4 --- /dev/null +++ b/fs_storage/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "Filesystem Storage Backend", + "summary": "Implement the concept of Storage with amazon S3, sftp...", + "version": "16.0.1.0.1", + "category": "FS Storage", + "website": "https://github.com/OCA/storage", + "author": " ACSONE SA/NV, Odoo Community Association (OCA)", + "license": "LGPL-3", + "development_status": "Beta", + "installable": True, + "depends": ["base", "base_sparse_field", "server_environment"], + "data": [ + "views/fs_storage_view.xml", + "security/ir.model.access.csv", + ], + "demo": ["demo/fs_storage.xml"], + "external_dependencies": {"python": ["fsspec"]}, +} diff --git a/fs_storage/demo/fs_storage.xml b/fs_storage/demo/fs_storage.xml new file mode 100644 index 0000000000..a1fd01fd70 --- /dev/null +++ b/fs_storage/demo/fs_storage.xml @@ -0,0 +1,8 @@ + + + + Odoo Filesystem Backend + odoofs + odoofs + + diff --git a/fs_storage/i18n/storage_backend.pot b/fs_storage/i18n/storage_backend.pot new file mode 100644 index 0000000000..45503812f9 --- /dev/null +++ b/fs_storage/i18n/storage_backend.pot @@ -0,0 +1,144 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fs_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type_env_default +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path_env_default +msgid " Env Default" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/components/filesystem_adapter.py:0 +#, python-format +msgid "Access to %s is forbidden" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type +msgid "Backend Type" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__backend_type_env_is_editable +msgid "Backend Type Env Is Editable" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Connection Test Failed!" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Connection Test Succeeded!" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__create_uid +msgid "Created by" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__create_date +msgid "Created on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path +msgid "Directory Path" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__directory_path_env_is_editable +msgid "Directory Path Env Is Editable" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__display_name +msgid "Display Name" +msgstr "" + +#. module: fs_storage +#. odoo-python +#: code:addons/fs_storage/models/fs_storage.py:0 +#, python-format +msgid "Everything seems properly set up!" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields.selection,name:fs_storage.selection__fs_storage__backend_type__filesystem +msgid "Filesystem" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__has_validation +msgid "Has Validation" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__id +msgid "ID" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage____last_update +msgid "Last Modified on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__write_uid +msgid "Last Updated by" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__write_date +msgid "Last Updated on" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__name +msgid "Name" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,help:fs_storage.field_fs_storage__directory_path +#: model:ir.model.fields,help:fs_storage.field_fs_storage__directory_path_env_default +msgid "Relative path to the directory to store the file" +msgstr "" + +#. module: fs_storage +#: model:ir.model.fields,field_description:fs_storage.field_fs_storage__server_env_defaults +msgid "Server Env Defaults" +msgstr "" + +#. module: fs_storage +#: model:ir.actions.act_window,name:fs_storage.act_open_fs_storage_view +#: model:ir.model,name:fs_storage.model_fs_storage +#: model:ir.ui.menu,name:fs_storage.menu_storage +#: model:ir.ui.menu,name:fs_storage.menu_fs_storage +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_form +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_search +msgid "FS Storage" +msgstr "" + +#. module: fs_storage +#: model_terms:ir.ui.view,arch_db:fs_storage.fs_storage_view_form +msgid "Test connection" +msgstr "" diff --git a/fs_storage/models/__init__.py b/fs_storage/models/__init__.py new file mode 100644 index 0000000000..349bb0495a --- /dev/null +++ b/fs_storage/models/__init__.py @@ -0,0 +1 @@ +from . import fs_storage diff --git a/fs_storage/models/fs_storage.py b/fs_storage/models/fs_storage.py new file mode 100644 index 0000000000..5bcc5f3e5e --- /dev/null +++ b/fs_storage/models/fs_storage.py @@ -0,0 +1,409 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from __future__ import annotations + +import base64 +import functools +import inspect +import json +import logging +import os.path +import re +import warnings +from typing import AnyStr + +import fsspec + +from odoo import _, api, fields, models, tools +from odoo.exceptions import ValidationError + +from odoo.addons.base_sparse_field.models.fields import Serialized + +_logger = logging.getLogger(__name__) + + +# TODO: useful for the whole OCA? +def deprecated(reason): + """Mark functions or classes as deprecated. + + Emit warning at execution. + + The @deprecated is used with a 'reason'. + + .. code-block:: python + + @deprecated("please, use another function") + def old_function(x, y): + pass + """ + + def decorator(func1): + + if inspect.isclass(func1): + fmt1 = "Call to deprecated class {name} ({reason})." + else: + fmt1 = "Call to deprecated function {name} ({reason})." + + @functools.wraps(func1) + def new_func1(*args, **kwargs): + warnings.simplefilter("always", DeprecationWarning) + warnings.warn( + fmt1.format(name=func1.__name__, reason=reason), + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) + return func1(*args, **kwargs) + + return new_func1 + + return decorator + + +class FSStorage(models.Model): + _name = "fs.storage" + _inherit = "server.env.mixin" + _description = "FS Storage" + + __slots__ = ("__fs", "__odoo_storage_path") + + def __init__(self, env, ids=(), prefetch_ids=()): + super().__init__(env, ids=ids, prefetch_ids=prefetch_ids) + self.__fs = None + self.__odoo_storage_path = None + + name = fields.Char(required=True) + code = fields.Char( + required=True, + help="Technical code used to identify the storage backend into the code." + "This code must be unique. This code is used for example to define the " + "storage backend to store the attachments via the configuration parameter " + "'ir_attachment.storage.force.database' when the module 'fs_attachment' " + "is installed.", + ) + protocol = fields.Selection( + selection="_get_protocols", + required=True, + default="odoofs", + help="The protocol used to access the content of filesystem.\n" + "This list is the one supported by the fsspec library (see " + "https://filesystem-spec.readthedocs.io/en/latest). A filesystem protocol" + "is added by default and refers to the odoo local filesystem.\n" + "Pay attention that according to the protocol, some options must be" + "provided through the options field.", + ) + protocol_descr = fields.Text( + compute="_compute_protocol_descr", + ) + options = fields.Text( + help="The options used to initialize the filesystem.\n" + "This is a JSON field that depends on the protocol used.\n" + "For example, for the sftp protocol, you can provide the following:\n" + "{\n" + " 'host': 'my.sftp.server',\n" + " 'ssh_kwrags': {\n" + " 'username': 'myuser',\n" + " 'password': 'mypassword',\n" + " 'port': 22,\n" + " }\n" + "}\n" + "For more information, please refer to the fsspec documentation:\n" + "https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations" + ) + + json_options = Serialized( + help="The options used to initialize the filesystem.\n", + compute="_compute_json_options", + inverse="_inverse_json_options", + ) + directory_path = fields.Char( + help="Relative path to the directory to store the file" + ) + + # the next fields are used to display documentation to help the user + # to configure the backend + options_protocol = fields.Selection( + string="Describes Protocol", + selection="_get_options_protocol", + default="odoofs", + help="The protocol used to access the content of filesystem.\n" + "This list is the one supported by the fsspec library (see " + "https://filesystem-spec.readthedocs.io/en/latest). A filesystem protocol" + "is added by default and refers to the odoo local filesystem.\n" + "Pay attention that according to the protocol, some options must be" + "provided through the options field.", + store=False, + ) + options_properties = fields.Text( + string="Available properties", + compute="_compute_options_properties", + store=False, + ) + + _sql_constraints = [ + ( + "code_uniq", + "unique(code)", + "The code must be unique", + ), + ] + + _server_env_section_name_field = "code" + + @property + def _server_env_fields(self): + return {"protocol": {}, "options": {}, "directory_path": {}} + + def write(self, vals): + self.__fs = None + self.clear_caches() + return super().write(vals) + + @api.model + @tools.ormcache() + def get_id_by_code_map(self): + """Return a dictionary with the code as key and the id as value.""" + return {rec.code: rec.id for rec in self.search([])} + + @api.model + def get_id_by_code(self, code): + """Return the id of the filesystem associated to the given code.""" + return self.get_id_by_code_map().get(code) + + @api.model + def get_by_code(self, code) -> FSStorage: + """Return the filesystem associated to the given code.""" + res = self.browse() + res_id = self.get_id_by_code(code) + if res_id: + res = self.browse(res_id) + return res + + @api.model + @tools.ormcache() + def get_storage_codes(self): + """Return the list of codes of the existing filesystems.""" + return [s.code for s in self.search([])] + + @api.model + @tools.ormcache("code") + def get_fs_by_code(self, code): + """Return the filesystem associated to the given code. + + :param code: the code of the filesystem + """ + fs = None + fs_storage = self.get_by_code(code) + if fs_storage: + fs = fs_storage.fs + return fs + + def copy(self, default=None): + default = default or {} + if "code" not in default: + default["code"] = "{}_copy".format(self.code) + return super().copy(default) + + @api.model + def _get_protocols(self) -> list[tuple[str, str]]: + protocol = [("odoofs", "Odoo's FileSystem")] + for p in fsspec.available_protocols(): + try: + cls = fsspec.get_filesystem_class(p) + protocol.append((p, f"{p} ({cls.__name__})")) + except ImportError as e: + _logger.debug("Cannot load the protocol %s. Reason: %s", p, e) + return protocol + + @api.constrains("options") + def _check_options(self) -> None: + for rec in self: + try: + json.loads(rec.options or "{}") + except Exception as e: + raise ValidationError(_("The options must be a valid JSON")) from e + + @api.depends("options") + def _compute_json_options(self) -> None: + for rec in self: + rec.json_options = json.loads(rec.options or "{}") + + def _inverse_json_options(self) -> None: + for rec in self: + rec.options = json.dumps(rec.json_options) + + @api.depends("protocol") + def _compute_protocol_descr(self) -> None: + for rec in self: + rec.protocol_descr = fsspec.get_filesystem_class(rec.protocol).__doc__ + + @api.model + def _get_options_protocol(self) -> list[tuple[str, str]]: + protocol = [("odoofs", "Odoo's Filesystem")] + for p in fsspec.available_protocols(): + try: + fsspec.get_filesystem_class(p) + protocol.append((p, p)) + except ImportError as e: + _logger.debug("Cannot load the protocol %s. Reason: %s", p, e) + return protocol + + @api.depends("options_protocol") + def _compute_options_properties(self) -> None: + for rec in self: + cls = fsspec.get_filesystem_class(rec.options_protocol) + signature = inspect.signature(cls.__init__) + doc = inspect.getdoc(cls.__init__) + rec.options_properties = f"__init__{signature}\n{doc}" + + @property + def fs(self) -> fsspec.AbstractFileSystem: + """Get the fsspec filesystem for this backend.""" + self.ensure_one() + if not self.__fs: + self.__fs = self._get_filesystem() + return self.__fs + + def _get_filesystem_storage_path(self) -> str: + """Get the path to the storage directory. + + This path is relative to the odoo filestore.and is used as root path + when the protocol is filesystem. + """ + self.ensure_one() + path = os.path.join(self.env["ir.attachment"]._filestore(), "storage") + if not os.path.exists(path): + os.makedirs(path) + return path + + @property + def _odoo_storage_path(self) -> str: + """Get the path to the storage directory. + + This path is relative to the odoo filestore.and is used as root path + when the protocol is filesystem. + """ + if not self.__odoo_storage_path: + self.__odoo_storage_path = self._get_filesystem_storage_path() + return self.__odoo_storage_path + + def _recursive_add_odoo_storage_path(self, options: dict) -> dict: + """Add the odoo storage path to the options. + + This is a recursive function that will add the odoo_storage_path + option to the nested target_options if the target_protocol is + odoofs + """ + if "target_protocol" in options: + target_options = options.get("target_options", {}) + if options["target_protocol"] == "odoofs": + target_options["odoo_storage_path"] = self._odoo_storage_path + options["target_options"] = target_options + self._recursive_add_odoo_storage_path(target_options) + return options + + def _get_filesystem(self) -> fsspec.AbstractFileSystem: + """Get the fsspec filesystem for this backend. + + See https://filesystem-spec.readthedocs.io/en/latest/api.html + #fsspec.spec.AbstractFileSystem + + :return: fsspec.AbstractFileSystem + """ + self.ensure_one() + options = self.json_options + if self.protocol == "odoofs": + options["odoo_storage_path"] = self._odoo_storage_path + options = self._recursive_add_odoo_storage_path(options) + fs = fsspec.filesystem(self.protocol, **options) + directory_path = self.directory_path + if directory_path: + fs = fsspec.filesystem("rooted_dir", path=directory_path, fs=fs) + return fs + + # Deprecated methods used to ease the migration from the storage_backend addons + # to the fs_storage addons. These methods will be removed in the future (Odoo 18) + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def add(self, relative_path, data, binary=True, **kwargs) -> None: + if not binary: + data = base64.b64decode(data) + path = relative_path.split(self.fs.sep)[:-1] + if not self.fs.exists(self.fs.sep.join(path)): + self.fs.makedirs(self.fs.sep.join(path)) + with self.fs.open(relative_path, "wb", **kwargs) as f: + f.write(data) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def get(self, relative_path, binary=True, **kwargs) -> AnyStr: + data = self.fs.read_bytes(relative_path, **kwargs) + if not binary and data: + data = base64.b64encode(data) + return data + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def list_files(self, relative_path="", pattern=False) -> list[str]: + relative_path = relative_path or self.fs.root_marker + if not self.fs.exists(relative_path): + return [] + if pattern: + relative_path = self.fs.sep.join([relative_path, pattern]) + return self.fs.glob(relative_path) + return self.fs.ls(relative_path, detail=False) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def find_files(self, pattern, relative_path="", **kw) -> list[str]: + """Find files matching given pattern. + + :param pattern: regex expression + :param relative_path: optional relative path containing files + :return: list of file paths as full paths from the root + """ + result = [] + relative_path = relative_path or self.fs.root_marker + if not self.fs.exists(relative_path): + return [] + regex = re.compile(pattern) + for file_path in self.fs.ls(relative_path, detail=False): + if regex.match(file_path): + result.append(file_path) + return result + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def move_files(self, files, destination_path, **kw) -> None: + """Move files to given destination. + + :param files: list of file paths to be moved + :param destination_path: directory path where to move files + :return: None + """ + for file_path in files: + self.fs.move( + file_path, + self.fs.sep.join([destination_path, os.path.basename(file_path)]), + **kw, + ) + + @deprecated("Please use _get_filesystem() instead and the fsspec API directly.") + def delete(self, relative_path) -> None: + self.fs.rm_file(relative_path) + + def action_test_config(self) -> None: + try: + self.fs.ls("", detail=False) + title = _("Connection Test Succeeded!") + message = _("Everything seems properly set up!") + msg_type = "success" + except Exception as err: + title = _("Connection Test Failed!") + message = str(err) + msg_type = "danger" + return { + "type": "ir.actions.client", + "tag": "display_notification", + "params": { + "title": title, + "message": message, + "type": msg_type, + "sticky": False, + }, + } diff --git a/fs_storage/odoo_file_system.py b/fs_storage/odoo_file_system.py new file mode 100644 index 0000000000..1827a31acc --- /dev/null +++ b/fs_storage/odoo_file_system.py @@ -0,0 +1,50 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + + +from fsspec.registry import register_implementation + +from .rooted_dir_file_system import RootedDirFileSystem + + +class OdooFileSystem(RootedDirFileSystem): + """A directory-based filesystem for Odoo. + + This filesystem is mounted from a specific subdirectory of the Odoo + filestore directory. + + It extends the RootedDirFileSystem to avoid going outside the + specific subdirectory nor the Odoo filestore directory. + + Parameters: + odoo_storage_path: The path of the subdirectory of the Odoo filestore + directory to mount. This parameter is required and is always provided + by the Odoo FS Storage even if it is explicitly defined in the + storage options. + fs: AbstractFileSystem + An instantiated filesystem to wrap. + target_protocol, target_options: + if fs is none, construct it from these + """ + + def __init__( + self, + *, + odoo_storage_path, + fs=None, + target_protocol=None, + target_options=None, + **storage_options + ): + if not odoo_storage_path: + raise ValueError("odoo_storage_path is required") + super().__init__( + path=odoo_storage_path, + fs=fs, + target_protocol=target_protocol, + target_options=target_options, + **storage_options + ) + + +register_implementation("odoofs", OdooFileSystem) diff --git a/fs_storage/readme/CONTRIBUTORS.rst b/fs_storage/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..60c32f1d3f --- /dev/null +++ b/fs_storage/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Laurent Mignon +* Sébastien BEAU diff --git a/fs_storage/readme/DESCRIPTION.rst b/fs_storage/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..1cfc5c1a00 --- /dev/null +++ b/fs_storage/readme/DESCRIPTION.rst @@ -0,0 +1,63 @@ +This addon is a technical addon that allows you to define filesystem like +storage for your data. It's used by other addons to store their data in a +transparent way into different kind of storages. + +Through the fs.storage record, you get access to an object that implements +the `fsspec.spec.AbstractFileSystem `_ interface and therefore give +you an unified interface to access your data whatever the storage protocol you +decide to use. + +The list of supported protocols depends on the installed fsspec implementations. +By default, the addon will install the following protocols: + +* LocalFileSystem +* MemoryFileSystem +* ZipFileSystem +* TarFileSystem +* FTPFileSystem +* CachingFileSystem +* WholeFileSystem +* SimplCacheFileSystem +* ReferenceFileSystem +* GenericFileSystem +* DirFileSystem +* DatabricksFileSystem +* GitHubFileSystem +* JupiterFileSystem +* OdooFileSystem + +The OdooFileSystem is the one that allows you to store your data into a directory +mounted into your Odoo's storage directory. This is the default FS Storage +when creating a new fs.storage record. + +Others protocols are available through the installation of additional +python packages: + +* DropboxDriveFileSystem -> `pip install fsspec[dropbox]` +* HTTPFileSystem -> `pip install fsspec[http]` +* HTTPSFileSystem -> `pip install fsspec[http]` +* GCSFileSystem -> `pip install fsspec[gcs]` +* GSFileSystem -> `pip install fsspec[gs]` +* GoogleDriveFileSystem -> `pip install gdrivefs` +* SFTPFileSystem -> `pip install fsspec[sftp]` +* HaddoopFileSystem -> `pip install fsspec[hdfs]` +* S3FileSystem -> `pip install fsspec[s3]` +* WandbFS -> `pip install wandbfs` +* OCIFileSystem -> `pip install fsspec[oci]` +* AsyncLocalFileSystem -> `pip install 'morefs[asynclocalfs]` +* AzureDatalakeFileSystem -> `pip install fsspec[adl]` +* AzureBlobFileSystem -> `pip install fsspec[abfs]` +* DaskWorkerFileSystem -> `pip install fsspec[dask]` +* GitFileSystem -> `pip install fsspec[git]` +* SMBFileSystem -> `pip install fsspec[smb]` +* LibArchiveFileSystem -> `pip install fsspec[libarchive]` +* OSSFileSystem -> `pip install ossfs` +* WebdavFileSystem -> `pip install webdav4` +* DVCFileSystem -> `pip install dvc` +* XRootDFileSystem -> `pip install fsspec-xrootd` + +This list of supported protocols is not exhaustive or could change in the future +depending on the fsspec releases. You can find more information about the +supported protocols on the `fsspec documentation +`_. diff --git a/fs_storage/readme/HISTORY.rst b/fs_storage/readme/HISTORY.rst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fs_storage/readme/ROADMAP.rst b/fs_storage/readme/ROADMAP.rst new file mode 100644 index 0000000000..0c697994d0 --- /dev/null +++ b/fs_storage/readme/ROADMAP.rst @@ -0,0 +1,9 @@ +* Transactions: fsspec comes with a transactional mechanism that once started, + gathers all the files created during the transaction, and if the transaction + is committed, moves them to their final locations. It would be useful to + bridge this with the transactional mechanism of odoo. This would allow to + ensure that all the files created during a transaction are either all + moved to their final locations, or all deleted if the transaction is rolled + back. This mechanism is only valid for files created during the transaction + by a call to the `open` method of the file system. It is not valid for others + operations, such as `rm`, `mv_file`, ... . diff --git a/fs_storage/readme/USAGE.rst b/fs_storage/readme/USAGE.rst new file mode 100644 index 0000000000..f65fb39af2 --- /dev/null +++ b/fs_storage/readme/USAGE.rst @@ -0,0 +1,84 @@ +Configuration +~~~~~~~~~~~~~ + +When you create a new backend, you must specify the following: + +* The name of the backend. This is the name that will be used to + identify the backend into Odoo +* The code of the backend. This code will identify the backend into the store_fname + field of the ir.attachment model. This code must be unique. It will be used + as scheme. example of the store_fname field: ``odoofs://abs34Tg11``. +* The protocol used by the backend. The protocol refers to the supported + protocols of the fsspec python package. +* A directory path. This is a root directory from which the filesystem will + be mounted. This directory must exist. +* The protocol options. These are the options that will be passed to the + fsspec python package when creating the filesystem. These options depend + on the protocol used and are described in the fsspec documentation. + +Some protocols defined in the fsspec package are wrappers around other +protocols. For example, the SimpleCacheFileSystem protocol is a wrapper +around any local filesystem protocol. In such cases, you must specify into the +protocol options the protocol to be wrapped and the options to be passed to +the wrapped protocol. + +For example, if you want to create a backend that uses the SimpleCacheFileSystem +protocol, after selecting the SimpleCacheFileSystem protocol, you must specify +the protocol options as follows: + +.. code-block:: python + + { + "directory_path": "/tmp/my_backend", + "target_protocol": "odoofs", + "target_options": {...}, + } + +In this example, the SimpleCacheFileSystem protocol will be used as a wrapper +around the odoofs protocol. + +Server Environment +~~~~~~~~~~~~~~~~~~ + +To ease the management of the filesystem storages configuration accross the different +environments, the configuration of the filesystem storages can be defined in +environment files or directly in the main configuration file. For example, the +configuration of a filesystem storage with the code `fsprod` can be provided in the +main configuration file as follows: + +.. code-block:: ini + + [fs_storage.fsprod] + protocol=s3 + options={"endpoint_url": "https://my_s3_server/", "key": "KEY", "secret": "SECRET"} + directory_path=my_bucket + +To work, a `storage.backend` record must exist with the code `fsprod` into the database. +In your configuration section, you can specify the value for the following fields: + +* `protocol` +* `options` +* `directory_path` + +Migration from storage_backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The fs_storage addon can be used to replace the storage_backend addon. (It has +been designed to be a drop-in replacement for the storage_backend addon). To +ease the migration, the `fs.storage` model defines the high-level methods +available in the storage_backend model. These methods are: + +* `add` +* `get` +* `list_files` +* `find_files` +* `move_files` +* `delete` + +These methods are wrappers around the methods of the `fsspec.AbstractFileSystem` +class (see https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem). +These methods are marked as deprecated and will be removed in a future version (V18) +of the addon. You should use the methods of the `fsspec.AbstractFileSystem` class +instead since they are more flexible and powerful. You can access the instance +of the `fsspec.AbstractFileSystem` class using the `fs` property of a `fs.storage` +record. diff --git a/fs_storage/rooted_dir_file_system.py b/fs_storage/rooted_dir_file_system.py new file mode 100644 index 0000000000..8fb401685f --- /dev/null +++ b/fs_storage/rooted_dir_file_system.py @@ -0,0 +1,37 @@ +# Copyright 2023 ACSONE SA/NV (https://www.acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import os + +from fsspec.implementations.dirfs import DirFileSystem +from fsspec.implementations.local import make_path_posix +from fsspec.registry import register_implementation + + +class RootedDirFileSystem(DirFileSystem): + """A directory-based filesystem that uses path as a root. + + The main purpose of this filesystem is to ensure that paths are always + a sub path of the initial path. IOW, it is not possible to go outside + the initial path. That's the only difference with the DirFileSystem provided + by fsspec. + + This one should be provided by fsspec itself. We should propose a PR. + """ + + def _join(self, path): + path = super()._join(path) + # Ensure that the path is a subpath of the root path by resolving + # any relative paths. + # Since the path separator is not always the same on all systems, + # we need to normalize the path separator. + path_posix = os.path.normpath(make_path_posix(path, self.sep)) + root_posix = os.path.normpath(make_path_posix(self.path)) + if not path_posix.startswith(root_posix): + raise PermissionError( + "Path %s is not a subpath of the root path %s" % (path, self.path) + ) + return path + + +register_implementation("rooted_dir", RootedDirFileSystem) diff --git a/fs_storage/security/ir.model.access.csv b/fs_storage/security/ir.model.access.csv new file mode 100644 index 0000000000..9c079b2eef --- /dev/null +++ b/fs_storage/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_fs_storage_edit,fs_storage edit,model_fs_storage,base.group_system,1,1,1,1 diff --git a/fs_storage/static/description/icon.png b/fs_storage/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/fs_storage/static/description/icon.png differ diff --git a/fs_storage/static/description/index.html b/fs_storage/static/description/index.html new file mode 100644 index 0000000000..e1730e320d --- /dev/null +++ b/fs_storage/static/description/index.html @@ -0,0 +1,563 @@ + + + + + + +Filesystem Storage Backend + + + +
+

Filesystem Storage Backend

+ + +

Beta License: LGPL-3 OCA/storage Translate me on Weblate Try me on Runbot

+

This addon is a technical addon that allows you to define filesystem like +storage for your data. It’s used by other addons to store their data in a +transparent way into different kind of storages.

+

Through the fs.storage record, you get access to an object that implements +the fsspec.spec.AbstractFileSystem interface and therefore give +you an unified interface to access your data whatever the storage protocol you +decide to use.

+

The list of supported protocols depends on the installed fsspec implementations. +By default, the addon will install the following protocols:

+
    +
  • LocalFileSystem
    • +
  • MemoryFileSystem
    • +
  • ZipFileSystem
    • +
  • TarFileSystem
    • +
  • FTPFileSystem
    • +
  • CachingFileSystem
    • +
  • WholeFileSystem
    • +
  • SimplCacheFileSystem
    • +
  • ReferenceFileSystem
    • +
  • GenericFileSystem
    • +
  • DirFileSystem
    • +
  • DatabricksFileSystem
    • +
  • GitHubFileSystem
    • +
  • JupiterFileSystem
    • +
  • OdooFileSystem
    • +
+

The OdooFileSystem is the one that allows you to store your data into a directory +mounted into your Odoo’s storage directory. This is the default FS Storage +when creating a new fs.storage record.

+

Others protocols are available through the installation of additional +python packages:

+
    +
  • DropboxDriveFileSystem -> pip install fsspec[dropbox]
    • +
  • HTTPFileSystem -> pip install fsspec[http]
    • +
  • HTTPSFileSystem -> pip install fsspec[http]
    • +
  • GCSFileSystem -> pip install fsspec[gcs]
    • +
  • GSFileSystem -> pip install fsspec[gs]
    • +
  • GoogleDriveFileSystem -> pip install gdrivefs
    • +
  • SFTPFileSystem -> pip install fsspec[sftp]
    • +
  • HaddoopFileSystem -> pip install fsspec[hdfs]
    • +
  • S3FileSystem -> pip install fsspec[s3]
    • +
  • WandbFS -> pip install wandbfs
    • +
  • OCIFileSystem -> pip install fsspec[oci]
    • +
  • AsyncLocalFileSystem -> pip install ‘morefs[asynclocalfs]
    • +
  • AzureDatalakeFileSystem -> pip install fsspec[adl]
    • +
  • AzureBlobFileSystem -> pip install fsspec[abfs]
    • +
  • DaskWorkerFileSystem -> pip install fsspec[dask]
    • +
  • GitFileSystem -> pip install fsspec[git]
    • +
  • SMBFileSystem -> pip install fsspec[smb]
    • +
  • LibArchiveFileSystem -> pip install fsspec[libarchive]
    • +
  • OSSFileSystem -> pip install ossfs
    • +
  • WebdavFileSystem -> pip install webdav4
    • +
  • DVCFileSystem -> pip install dvc
    • +
  • XRootDFileSystem -> pip install fsspec-xrootd
    • +
+

This list of supported protocols is not exhaustive or could change in the future +depending on the fsspec releases. You can find more information about the +supported protocols on the fsspec documentation.

+

Table of contents

+ +
+

Usage

+
+

Configuration

+

When you create a new backend, you must specify the following:

+
    +
  • The name of the backend. This is the name that will be used to +identify the backend into Odoo
    • +
  • The code of the backend. This code will identify the backend into the store_fname +field of the ir.attachment model. This code must be unique. It will be used +as scheme. example of the store_fname field: odoofs://abs34Tg11.
    • +
  • The protocol used by the backend. The protocol refers to the supported +protocols of the fsspec python package.
    • +
  • A directory path. This is a root directory from which the filesystem will +be mounted. This directory must exist.
    • +
  • The protocol options. These are the options that will be passed to the +fsspec python package when creating the filesystem. These options depend +on the protocol used and are described in the fsspec documentation.
    • +
+

Some protocols defined in the fsspec package are wrappers around other +protocols. For example, the SimpleCacheFileSystem protocol is a wrapper +around any local filesystem protocol. In such cases, you must specify into the +protocol options the protocol to be wrapped and the options to be passed to +the wrapped protocol.

+

For example, if you want to create a backend that uses the SimpleCacheFileSystem +protocol, after selecting the SimpleCacheFileSystem protocol, you must specify +the protocol options as follows:

+
+{
+    "directory_path": "/tmp/my_backend",
+    "target_protocol": "odoofs",
+    "target_options": {...},
+}
+
+

In this example, the SimpleCacheFileSystem protocol will be used as a wrapper +around the odoofs protocol.

+
+
+

Server Environment

+

To ease the management of the filesystem storages configuration accross the different +environments, the configuration of the filesystem storages can be defined in +environment files or directly in the main configuration file. For example, the +configuration of a filesystem storage with the code fsprod can be provided in the +main configuration file as follows:

+
+[fs_storage.fsprod]
+protocol=s3
+options={"endpoint_url": "https://my_s3_server/", "key": "KEY", "secret": "SECRET"}
+directory_path=my_bucket
+
+

To work, a storage.backend record must exist with the code fsprod into the database. +In your configuration section, you can specify the value for the following fields:

+
    +
  • protocol
    • +
  • options
    • +
  • directory_path
    • +
+
+
+
+

Known issues / Roadmap

+
    +
  • Transactions: fsspec comes with a transactional mechanism that once started, +gathers all the files created during the transaction, and if the transaction +is committed, moves them to their final locations. It would be useful to +bridge this with the transactional mechanism of odoo. This would allow to +ensure that all the files created during a transaction are either all +moved to their final locations, or all deleted if the transaction is rolled +back. This mechanism is only valid for files created during the transaction +by a call to the open method of the file system. It is not valid for others +operations, such as rm, mv_file, … .
    • +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
  • ACSONE SA/NV
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/storage project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/fs_storage/tests/__init__.py b/fs_storage/tests/__init__.py new file mode 100644 index 0000000000..84bbda192b --- /dev/null +++ b/fs_storage/tests/__init__.py @@ -0,0 +1 @@ +from . import test_fs_storage diff --git a/fs_storage/tests/test_fs_storage.py b/fs_storage/tests/test_fs_storage.py new file mode 100644 index 0000000000..74dfde8814 --- /dev/null +++ b/fs_storage/tests/test_fs_storage.py @@ -0,0 +1,153 @@ +# Copyright 2023 ACSONE SA/NV (http://acsone.eu). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +import base64 +import shutil +import tempfile +import warnings +from unittest import mock + +from odoo.tests.common import TransactionCase +from odoo.tools import mute_logger + +from ..models.fs_storage import FSStorage + + +class TestFSStorage(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + cls.backend: FSStorage = cls.env.ref("fs_storage.default_fs_storage") + cls.backend.json_options = {"target_options": {"auto_mkdir": "True"}} + cls.filedata = base64.b64encode(b"This is a simple file") + cls.filename = "test_file.txt" + cls.case_with_subdirectory = "subdirectory/here" + cls.demo_user = cls.env.ref("base.user_demo") + + def setUp(self): + super().setUp() + mocked_backend = mock.patch.object( + self.backend.__class__, "_get_filesystem_storage_path" + ) + mocked_get_filesystem_storage_path = mocked_backend.start() + tempdir = tempfile.mkdtemp() + mocked_get_filesystem_storage_path.return_value = tempdir + + # pylint: disable=unused-variable + @self.addCleanup + def stop_mock(): + mocked_backend.stop() + # recursively delete the tempdir + shutil.rmtree(tempdir) + + def _create_file(self, backend: FSStorage, filename: str, filedata: str): + with backend.fs.open(filename, "wb") as f: + f.write(filedata) + + @mute_logger("py.warnings") + def _test_deprecated_setting_and_getting_data(self): + # Check that the directory is empty + warnings.filterwarnings("ignore") + files = self.backend.list_files() + self.assertNotIn(self.filename, files) + + # Add a new file + self.backend.add( + self.filename, self.filedata, mimetype="text/plain", binary=False + ) + + # Check that the file exist + files = self.backend.list_files() + self.assertIn(self.filename, files) + + # Retrieve the file added + data = self.backend.get(self.filename, binary=False) + self.assertEqual(data, self.filedata) + + # Delete the file + self.backend.delete(self.filename) + files = self.backend.list_files() + self.assertNotIn(self.filename, files) + + @mute_logger("py.warnings") + def _test_deprecated_find_files(self): + warnings.filterwarnings("ignore") + self.backend.add( + self.filename, self.filedata, mimetype="text/plain", binary=False + ) + try: + res = self.backend.find_files(r".*\.txt") + self.assertListEqual([self.filename], res) + res = self.backend.find_files(r".*\.text") + self.assertListEqual([], res) + finally: + self.backend.delete(self.filename) + + def test_deprecated_setting_and_getting_data_from_root(self): + self._test_deprecated_setting_and_getting_data() + + def test_deprecated_setting_and_getting_data_from_dir(self): + self.backend.directory_path = self.case_with_subdirectory + self._test_deprecated_setting_and_getting_data() + + def test_deprecated_find_files_from_root(self): + self._test_deprecated_find_files() + + def test_deprecated_find_files_from_dir(self): + self.backend.directory_path = self.case_with_subdirectory + self._test_deprecated_find_files() + + def test_ensure_one_fs_by_record(self): + # in this test we ensure that we've one fs by record + backend_ids = [] + for i in range(4): + backend_ids.append( + self.backend.create( + {"name": f"name{i}", "directory_path": f"{i}", "code": f"code{i}"} + ).id + ) + records = self.backend.browse(backend_ids) + fs = None + for rec in records: + self.assertNotEqual(fs, rec.fs) + + def test_relative_access(self): + self.backend.directory_path = self.case_with_subdirectory + self._create_file(self.backend, self.filename, self.filedata) + other_subdirectory = "other_subdirectory" + backend2 = self.backend.copy({"directory_path": other_subdirectory}) + self._create_file(backend2, self.filename, self.filedata) + with self.assertRaises(PermissionError), self.env.cr.savepoint(): + # check that we can't access outside the subdirectory + backend2.fs.ls("../") + with self.assertRaises(PermissionError), self.env.cr.savepoint(): + # check that we can't access the file into another subdirectory + backend2.fs.ls(f"../{self.case_with_subdirectory}") + self.backend.fs.rm_file(self.filename) + backend2.fs.rm_file(self.filename) + + def test_recursive_add_odoo_storage_path_to_options(self): + options = { + "directory_path": "/tmp/my_backend", + "target_protocol": "odoofs", + } + self.backend._recursive_add_odoo_storage_path(options) + self.assertEqual( + self.backend._odoo_storage_path, + options.get("target_options").get("odoo_storage_path"), + ) + options = { + "directory_path": "/tmp/my_backend", + "target_protocol": "dir", + "target_options": { + "path": "/my_backend", + "target_protocol": "odoofs", + }, + } + self.backend._recursive_add_odoo_storage_path(options) + self.assertEqual( + self.backend._odoo_storage_path, + options.get("target_options") + .get("target_options") + .get("odoo_storage_path"), + ) diff --git a/fs_storage/views/fs_storage_view.xml b/fs_storage/views/fs_storage_view.xml new file mode 100644 index 0000000000..3027f29152 --- /dev/null +++ b/fs_storage/views/fs_storage_view.xml @@ -0,0 +1,117 @@ + + + + fs.storage.tree (in fs_storage) + fs.storage + + + + + + + + + + fs.storage.form (in fs_storage) + fs.storage + +
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + fs.storage.search (in fs_storage) + fs.storage + + + + + + + + + FS Storage + ir.actions.act_window + fs.storage + tree,form + + [] + {} + + + + + form + + + + + + tree + + + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..73bcbf6d02 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# generated from manifests external_dependencies +fsspec diff --git a/setup/fs_storage/odoo/addons/fs_storage b/setup/fs_storage/odoo/addons/fs_storage new file mode 120000 index 0000000000..fb821ff7a8 --- /dev/null +++ b/setup/fs_storage/odoo/addons/fs_storage @@ -0,0 +1 @@ +../../../../fs_storage \ No newline at end of file diff --git a/setup/fs_storage/setup.py b/setup/fs_storage/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/fs_storage/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)