diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..a5e43536f3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# generated from manifests external_dependencies +fastexcel +polars diff --git a/sheet_dataframe_process/README.rst b/sheet_dataframe_process/README.rst new file mode 100644 index 0000000000..8fe6e0cc2b --- /dev/null +++ b/sheet_dataframe_process/README.rst @@ -0,0 +1,106 @@ +======================= +Sheet Dataframe Process +======================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github + :target: https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process + :alt: OCA/reporting-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-sheet_dataframe_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Based on an imported spreadsheet like file, this module allows to create +Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database + but in memory: you don't need to iterate on each line to perform + operations +- the operations are powerful: filter, add column resulting from + calculation , select a subset of data + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/sheet_dataframe_process/__init__.py b/sheet_dataframe_process/__init__.py new file mode 100644 index 0000000000..aee8895e7a --- /dev/null +++ b/sheet_dataframe_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/sheet_dataframe_process/__manifest__.py b/sheet_dataframe_process/__manifest__.py new file mode 100644 index 0000000000..ea2bd94104 --- /dev/null +++ b/sheet_dataframe_process/__manifest__.py @@ -0,0 +1,34 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Sheet Dataframe Process", + "version": "18.0.1.0.0", + "category": "Reporting", + "license": "AGPL-3", + "development_status": "Alpha", + "summary": "Allow to create a Polars dataframe from a sheet file and " + "process it according to rules", + "website": "https://github.com/OCA/reporting-engine", + "maintainers": ["bealdav"], + "depends": [ + "contacts", + ], + "external_dependencies": { + "python": [ + "polars", + "fastexcel", + ] + }, + "data": [ + "data/action.xml", + "data/demo.xml", + "security/ir.model.access.xml", + "wizards/sheet_dataframe.xml", + "views/file_config.xml", + "views/file_field.xml", + "views/file_partner_field.xml", + "views/test_polars_file.xml", + "views/menu.xml", + ], + "installable": True, +} diff --git a/sheet_dataframe_process/data/action.xml b/sheet_dataframe_process/data/action.xml new file mode 100644 index 0000000000..a11c461d1a --- /dev/null +++ b/sheet_dataframe_process/data/action.xml @@ -0,0 +1,20 @@ + + + + + Update config + + + code + env["file.config"]._refresh_conf_hook() + + + + 🐻‍❄️ Populate file polars example + + + code + env["test.polars.file"]._populate() + + + diff --git a/sheet_dataframe_process/data/demo.xml b/sheet_dataframe_process/data/demo.xml new file mode 100644 index 0000000000..ca098832d1 --- /dev/null +++ b/sheet_dataframe_process/data/demo.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + skip + + + diff --git a/sheet_dataframe_process/models/__init__.py b/sheet_dataframe_process/models/__init__.py new file mode 100644 index 0000000000..6f595435f3 --- /dev/null +++ b/sheet_dataframe_process/models/__init__.py @@ -0,0 +1,4 @@ +from . import file_config +from . import file_field +from . import file_partner_field +from . import test_file diff --git a/sheet_dataframe_process/models/file_config.py b/sheet_dataframe_process/models/file_config.py new file mode 100644 index 0000000000..bf08a3629d --- /dev/null +++ b/sheet_dataframe_process/models/file_config.py @@ -0,0 +1,51 @@ +from odoo import Command, fields, models + + +class FileConfig(models.Model): + _name = "file.config" + _inherit = "mail.thread" + _description = "File Configuration" + _rec_name = "model_id" + + model_id = fields.Many2one( + comodel_name="ir.model", required=True, ondelete="cascade" + ) + code = fields.Char(help="Allow to browse between several identical models") + action = fields.Selection( + selection=[ + ("display", "Display"), + ("dataframe", "Dataframe"), + ], + default="display", + help="Some other behaviors can be implemented", + ) + partner_ids = fields.Many2many( + comodel_name="res.partner", domain="[('active', 'in', (True, False))]" + ) + field_ids = fields.One2many( + comodel_name="file.field", inverse_name="config_id", copy=True + ) + field_match_ids = fields.One2many( + comodel_name="file.partner.field", inverse_name="config_id", copy=True + ) + + def populate_match_lines(self): + # TODO use api depends instead of ui button + self.ensure_one() + for partner in self.partner_ids: + ffields = self.field_match_ids.filtered( + lambda s, partner=partner: s.partner_id == partner + ).mapped("field_id") + line_ids = self.field_ids.filtered( + lambda s, ffields=ffields: s.field_id not in ffields + ).mapped("id") + self.field_match_ids = [ + Command.create({"partner_id": partner.id, "line_id": x}) + for x in line_ids + ] + self.field_match_ids.filtered( + lambda s: s.partner_id not in s.config_id.partner_ids + ).unlink() + + def _refresh_conf_hook(self): + "You use it to trigger specific behavior" diff --git a/sheet_dataframe_process/models/file_field.py b/sheet_dataframe_process/models/file_field.py new file mode 100644 index 0000000000..319578cab1 --- /dev/null +++ b/sheet_dataframe_process/models/file_field.py @@ -0,0 +1,31 @@ +from odoo import fields, models + + +class FileField(models.Model): + _name = "file.field" + _description = "Configuration de l'import de champ" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade" + ) + field_id = fields.Many2one( + comodel_name="ir.model.fields", + ondelete="cascade", + required=True, + domain="[('model_id', '=', model_id)]", + # [('model_id', '=', model_id)] + ) + model_id = fields.Many2one( + comodel_name="ir.model", + readonly=True, + ) + required = fields.Boolean( + tracking=True, + help="Prevent to import missing data if field is missing in some records", + ) + on_fail = fields.Selection( + selection=[("stop", "Stop Process"), ("skip", "Skip record (TODO)")], + default="stop", + help="What should be the behavior in case of failure regarding constraint " + "fields (required, format, etc)", + ) diff --git a/sheet_dataframe_process/models/file_partner_field.py b/sheet_dataframe_process/models/file_partner_field.py new file mode 100644 index 0000000000..c1145747cb --- /dev/null +++ b/sheet_dataframe_process/models/file_partner_field.py @@ -0,0 +1,15 @@ +from odoo import fields, models + + +class FilePartnerField(models.Model): + _name = "file.partner.field" + _inherits = {"file.field": "line_id"} + _description = "Configuration de l'import de champ" + + line_id = fields.Many2one( + comodel_name="file.field", required=True, ondelete="cascade" + ) + partner_id = fields.Many2one( + comodel_name="res.partner", + ) + matching_column = fields.Char(help="Field name in spreadsheet") diff --git a/sheet_dataframe_process/models/test_file.py b/sheet_dataframe_process/models/test_file.py new file mode 100644 index 0000000000..71207c3c84 --- /dev/null +++ b/sheet_dataframe_process/models/test_file.py @@ -0,0 +1,85 @@ +import base64 +from pathlib import Path + +from odoo import fields, models +from odoo.modules.module import get_module_path + + +class TestPolarsFile(models.Model): + _name = "test.polars.file" + _description = "Example files to ensure your configuration match with cases" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade", readonly=True + ) + name = fields.Char() + template = fields.Binary(string="Fichier", attachment=False) + + def _populate(self): + def create_attach(myfile, addon, idstring, relative_path): + with open(myfile, "rb") as f: + vals = { + "config_id": self.env.ref(idstring).id, + "name": f.name[f.name.find(addon) :], + } + self.env[self._name].sudo().create(vals) + + self.env[self._name].search([("template", "=", False)]).unlink() + paths = self._get_test_file_paths() + for addon, data in paths.items(): + relative_path = data["relative_path"] + idstring = f"{addon}.{data['xmlid']}" + if self.env.ref(idstring): + mpath = Path(get_module_path(addon)) / relative_path + for mfile in tuple(mpath.iterdir()): + create_attach(mfile, addon, idstring, relative_path) + action = self.env.ref( + "sheet_dataframe_process.test_polars_file_action" + )._get_action_dict() + return action + + def try_import(self): + self.ensure_one() + transient = self.env["sheet.dataframe.transient"].create( + { + "filename": self.name, + "file": self._get_file(), + "config_id": self.config_id.id, + } + ) + action = self.env.ref( + "sheet_dataframe_process.sheet_dataframe_transient_action" + )._get_action_dict() + action["res_id"] = transient.id + return action + + def _get_file(self): + # TODO Clean + if self.template: + return self.template + module = self.name[: self.name.find("/")] + relative = self._get_test_file_paths().get(module) + relative = relative and relative.get("relative_path") + if relative: + path = Path(get_module_path(module)) + path = path / relative / self.name[self.name.rfind("/") + 1 :] + # myfile = path / self.name + with open(path, "rb") as f: + return base64.b64encode(f.read()) + + def _get_test_file_paths(self): + """ + You may override if you want populate files in your module + returns: + {"module_name": { + "relative_path": "tests/files", + "xmlid": "file_config_xml_id"} + } + } + """ + return { + "sheet_dataframe_process": { + "relative_path": "tests/files", + "xmlid": "file_config_contact", + } + } diff --git a/sheet_dataframe_process/pyproject.toml b/sheet_dataframe_process/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/sheet_dataframe_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/sheet_dataframe_process/readme/DESCRIPTION.md b/sheet_dataframe_process/readme/DESCRIPTION.md new file mode 100644 index 0000000000..80640f33fe --- /dev/null +++ b/sheet_dataframe_process/readme/DESCRIPTION.md @@ -0,0 +1,21 @@ +Based on an imported spreadsheet like file, this module allows to create +Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update + + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database but in memory: you don't need to iterate on each line to perform operations +- the operations are powerful: filter, add column resulting from calculation , select a subset of data + + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project diff --git a/sheet_dataframe_process/security/ir.model.access.xml b/sheet_dataframe_process/security/ir.model.access.xml new file mode 100644 index 0000000000..c8b3b607f2 --- /dev/null +++ b/sheet_dataframe_process/security/ir.model.access.xml @@ -0,0 +1,53 @@ + + + + File Config + + + + + + + + + + File Field + + + + + + + + + + File Partner Field + + + + + + + + + + sheet.dataframe.transient + + + + + + + + + + test polars file + + + + + + + + + diff --git a/sheet_dataframe_process/static/description/icon.png b/sheet_dataframe_process/static/description/icon.png new file mode 100644 index 0000000000..d1d2b57297 Binary files /dev/null and b/sheet_dataframe_process/static/description/icon.png differ diff --git a/sheet_dataframe_process/static/description/index.html b/sheet_dataframe_process/static/description/index.html new file mode 100644 index 0000000000..e100d32c71 --- /dev/null +++ b/sheet_dataframe_process/static/description/index.html @@ -0,0 +1,446 @@ + + + + + +Sheet Dataframe Process + + + +
+

Sheet Dataframe Process

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

Based on an imported spreadsheet like file, this module allows to create +Polars dataframe and process them according to rules in order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update

+

Why dataframe ?

+
    +
  • a dataframe is a kind of in-memory dataset on which you can operate
    • +
  • you can operates on your entire dataset a bit like with a database +but in memory: you don’t need to iterate on each line to perform +operations
    • +
  • the operations are powerful: filter, add column resulting from +calculation , select a subset of data
    • +
+

Why Polars ?

+
    +
  • performance: code in rust
    • +
  • environment consideration
    • +
  • dynamic project
    • +
+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/sheet_dataframe_process/tests/__init__.py b/sheet_dataframe_process/tests/__init__.py new file mode 100644 index 0000000000..d9b96c4fa5 --- /dev/null +++ b/sheet_dataframe_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/sheet_dataframe_process/tests/files/contact_odoo.xlsx b/sheet_dataframe_process/tests/files/contact_odoo.xlsx new file mode 100644 index 0000000000..6251f6e78d Binary files /dev/null and b/sheet_dataframe_process/tests/files/contact_odoo.xlsx differ diff --git a/sheet_dataframe_process/tests/test_module.py b/sheet_dataframe_process/tests/test_module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sheet_dataframe_process/views/file_config.xml b/sheet_dataframe_process/views/file_config.xml new file mode 100644 index 0000000000..648daea2bb --- /dev/null +++ b/sheet_dataframe_process/views/file_config.xml @@ -0,0 +1,90 @@ + + + + file.config + +
+
+
+ + + + + + + + + + + + + + + + + + +
+ + + + file.config + + + + + + + + + + + + file.config + + + + + + + + + + + + + + + Configuration + file.config + list,form + file-config + + + diff --git a/sheet_dataframe_process/views/file_field.xml b/sheet_dataframe_process/views/file_field.xml new file mode 100644 index 0000000000..43cc4dc759 --- /dev/null +++ b/sheet_dataframe_process/views/file_field.xml @@ -0,0 +1,26 @@ + + + + file.field + + + + + + + + + + + + File Field + file.field + list + file-field + + + diff --git a/sheet_dataframe_process/views/file_partner_field.xml b/sheet_dataframe_process/views/file_partner_field.xml new file mode 100644 index 0000000000..4a85c1f058 --- /dev/null +++ b/sheet_dataframe_process/views/file_partner_field.xml @@ -0,0 +1,26 @@ + + + + file.partner.field + + + + + + + + + + + + File Partner Field + file.partner.field + list + file-partner-field + + + diff --git a/sheet_dataframe_process/views/menu.xml b/sheet_dataframe_process/views/menu.xml new file mode 100644 index 0000000000..4db9413502 --- /dev/null +++ b/sheet_dataframe_process/views/menu.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + diff --git a/sheet_dataframe_process/views/test_polars_file.xml b/sheet_dataframe_process/views/test_polars_file.xml new file mode 100644 index 0000000000..9b99a4b549 --- /dev/null +++ b/sheet_dataframe_process/views/test_polars_file.xml @@ -0,0 +1,28 @@ + + + + + test.polars.file + + + +