[ADD] sheet_to_dataframe: convert file to polars dataframe and proces…

…s it

requirements.txt

@@ -0,0 +1,3 @@
+# generated from manifests external_dependencies
+fastexcel
+polars

sheet_dataframe_process/README.rst

@@ -0,0 +1,106 @@
+=======================
+Sheet Dataframe Process
+=======================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github
+    :target: https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process
+    :alt: OCA/reporting-engine
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-sheet_dataframe_process
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+Based on an imported spreadsheet like file, this module allows to create
+Polars dataframe and process them according to rules in order to:
+
+-  filter data and display
+-  obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to
+create/update
+
+Why dataframe ?
+
+-  a dataframe is a kind of in-memory dataset on which you can operate
+-  you can operates on your entire dataset a bit like with a database
+   but in memory: you don't need to iterate on each line to perform
+   operations
+-  the operations are powerful: filter, add column resulting from
+   calculation , select a subset of data
+
+Why Polars ?
+
+-  performance: code in rust
+-  environment consideration
+-  dynamic project
+
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/reporting-engine/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/reporting-engine/issues/new?body=module:%20sheet_dataframe_process%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* 
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px
+    :target: https://github.com/bealdav
+    :alt: bealdav
+
+Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-bealdav| 
+
+This module is part of the `OCA/reporting-engine <https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

sheet_dataframe_process/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import wizards

sheet_dataframe_process/__manifest__.py

@@ -0,0 +1,34 @@
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Sheet Dataframe Process",
+    "version": "18.0.1.0.0",
+    "category": "Reporting",
+    "license": "AGPL-3",
+    "development_status": "Alpha",
+    "summary": "Allow to create a Polars dataframe from a sheet file and "
+    "process it according to rules",
+    "website": "https://github.com/OCA/reporting-engine",
+    "maintainers": ["bealdav"],
+    "depends": [
+        "contacts",
+    ],
+    "external_dependencies": {
+        "python": [
+            "polars",
+            "fastexcel",
+        ]
+    },
+    "data": [
+        "data/action.xml",
+        "data/demo.xml",
+        "security/ir.model.access.xml",
+        "wizards/sheet_dataframe.xml",
+        "views/file_config.xml",
+        "views/file_field.xml",
+        "views/file_partner_field.xml",
+        "views/test_polars_file.xml",
+        "views/menu.xml",
+    ],
+    "installable": True,
+}

sheet_dataframe_process/data/action.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+
+<record id="refresh_conf" model="ir.actions.server">
+    <field name="name">Update config</field>
+    <field name="model_id" ref="model_file_config" />
+    <field name="binding_model_id" ref="model_file_config" />
+    <field name="state">code</field>
+    <field name="code">env["file.config"]._refresh_conf_hook()</field>
+</record>
+
+<record id="refresh_file_example" model="ir.actions.server">
+    <field name="name">🐻‍❄️ Populate file polars example</field>
+    <field name="model_id" ref="model_test_polars_file" />
+    <field name="binding_model_id" ref="model_test_polars_file" />
+    <field name="state">code</field>
+    <field name="code">env["test.polars.file"]._populate()</field>
+</record>
+
+</odoo>

sheet_dataframe_process/data/demo.xml

@@ -0,0 +1,22 @@
+<odoo>
+
+    <record id="file_config_contact" model="file.config">
+        <field name="model_id" ref="base.model_res_partner" />
+        <field
+            name="partner_ids"
+            eval="[ref('base.res_partner_18'), ref('base.res_partner_5')]"
+        />
+    </record>
+
+    <record id="file_config_contact_country" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__country_code" />
+    </record>
+    <record id="file_config_contact_name" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__name" />
+        <field name="required" eval="1" />
+        <field name="on_fail">skip</field>
+    </record>
+
+</odoo>

sheet_dataframe_process/models/__init__.py

@@ -0,0 +1,4 @@
+from . import file_config
+from . import file_field
+from . import file_partner_field
+from . import test_file

sheet_dataframe_process/models/file_config.py

@@ -0,0 +1,51 @@
+from odoo import Command, fields, models
+
+
+class FileConfig(models.Model):
+    _name = "file.config"
+    _inherit = "mail.thread"
+    _description = "File Configuration"
+    _rec_name = "model_id"
+
+    model_id = fields.Many2one(
+        comodel_name="ir.model", required=True, ondelete="cascade"
+    )
+    code = fields.Char(help="Allow to browse between several identical models")
+    action = fields.Selection(
+        selection=[
+            ("display", "Display"),
+            ("dataframe", "Dataframe"),
+        ],
+        default="display",
+        help="Some other behaviors can be implemented",
+    )
+    partner_ids = fields.Many2many(
+        comodel_name="res.partner", domain="[('active', 'in', (True, False))]"
+    )
+    field_ids = fields.One2many(
+        comodel_name="file.field", inverse_name="config_id", copy=True
+    )
+    field_match_ids = fields.One2many(
+        comodel_name="file.partner.field", inverse_name="config_id", copy=True
+    )
+
+    def populate_match_lines(self):
+        # TODO use api depends instead of ui button
+        self.ensure_one()
+        for partner in self.partner_ids:
+            ffields = self.field_match_ids.filtered(
+                lambda s, partner=partner: s.partner_id == partner
+            ).mapped("field_id")
+            line_ids = self.field_ids.filtered(
+                lambda s, ffields=ffields: s.field_id not in ffields
+            ).mapped("id")
+            self.field_match_ids = [
+                Command.create({"partner_id": partner.id, "line_id": x})
+                for x in line_ids
+            ]
+        self.field_match_ids.filtered(
+            lambda s: s.partner_id not in s.config_id.partner_ids
+        ).unlink()
+
+    def _refresh_conf_hook(self):
+        "You use it to trigger specific behavior"

sheet_dataframe_process/models/file_field.py

@@ -0,0 +1,31 @@
+from odoo import fields, models
+
+
+class FileField(models.Model):
+    _name = "file.field"
+    _description = "Configuration de l'import de champ"
+
+    config_id = fields.Many2one(
+        comodel_name="file.config", required=True, ondelete="cascade"
+    )
+    field_id = fields.Many2one(
+        comodel_name="ir.model.fields",
+        ondelete="cascade",
+        required=True,
+        domain="[('model_id', '=', model_id)]",
+        # [('model_id', '=', model_id)]
+    )
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        readonly=True,
+    )
+    required = fields.Boolean(
+        tracking=True,
+        help="Prevent to import missing data if field is missing in some records",
+    )
+    on_fail = fields.Selection(
+        selection=[("stop", "Stop Process"), ("skip", "Skip record (TODO)")],
+        default="stop",
+        help="What should be the behavior in case of failure regarding constraint "
+        "fields (required, format, etc)",
+    )

sheet_dataframe_process/models/file_partner_field.py

@@ -0,0 +1,15 @@
+from odoo import fields, models
+
+
+class FilePartnerField(models.Model):
+    _name = "file.partner.field"
+    _inherits = {"file.field": "line_id"}
+    _description = "Configuration de l'import de champ"
+
+    line_id = fields.Many2one(
+        comodel_name="file.field", required=True, ondelete="cascade"
+    )
+    partner_id = fields.Many2one(
+        comodel_name="res.partner",
+    )
+    matching_column = fields.Char(help="Field name in spreadsheet")

sheet_dataframe_process/models/test_file.py

@@ -0,0 +1,85 @@
+import base64
+from pathlib import Path
+
+from odoo import fields, models
+from odoo.modules.module import get_module_path
+
+
+class TestPolarsFile(models.Model):
+    _name = "test.polars.file"
+    _description = "Example files to ensure your configuration match with cases"
+
+    config_id = fields.Many2one(
+        comodel_name="file.config", required=True, ondelete="cascade", readonly=True
+    )
+    name = fields.Char()
+    template = fields.Binary(string="Fichier", attachment=False)
+
+    def _populate(self):
+        def create_attach(myfile, addon, idstring, relative_path):
+            with open(myfile, "rb") as f:
+                vals = {
+                    "config_id": self.env.ref(idstring).id,
+                    "name": f.name[f.name.find(addon) :],
+                }
+                self.env[self._name].sudo().create(vals)
+
+        self.env[self._name].search([("template", "=", False)]).unlink()
+        paths = self._get_test_file_paths()
+        for addon, data in paths.items():
+            relative_path = data["relative_path"]
+            idstring = f"{addon}.{data['xmlid']}"
+            if self.env.ref(idstring):
+                mpath = Path(get_module_path(addon)) / relative_path
+                for mfile in tuple(mpath.iterdir()):
+                    create_attach(mfile, addon, idstring, relative_path)
+        action = self.env.ref(
+            "sheet_dataframe_process.test_polars_file_action"
+        )._get_action_dict()
+        return action
+
+    def try_import(self):
+        self.ensure_one()
+        transient = self.env["sheet.dataframe.transient"].create(
+            {
+                "filename": self.name,
+                "file": self._get_file(),
+                "config_id": self.config_id.id,
+            }
+        )
+        action = self.env.ref(
+            "sheet_dataframe_process.sheet_dataframe_transient_action"
+        )._get_action_dict()
+        action["res_id"] = transient.id
+        return action
+
+    def _get_file(self):
+        # TODO Clean
+        if self.template:
+            return self.template
+        module = self.name[: self.name.find("/")]
+        relative = self._get_test_file_paths().get(module)
+        relative = relative and relative.get("relative_path")
+        if relative:
+            path = Path(get_module_path(module))
+            path = path / relative / self.name[self.name.rfind("/") + 1 :]
+            # myfile = path / self.name
+            with open(path, "rb") as f:
+                return base64.b64encode(f.read())
+
+    def _get_test_file_paths(self):
+        """
+        You may override if you want populate files in your module
+        returns:
+        {"module_name": {
+            "relative_path": "tests/files",
+            "xmlid": "file_config_xml_id"}
+            }
+        }
+        """
+        return {
+            "sheet_dataframe_process": {
+                "relative_path": "tests/files",
+                "xmlid": "file_config_contact",
+            }
+        }

sheet_dataframe_process/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"