Release v1.10+ (#175)

* style: updated pre commit to latest versions (#159) style: update pre-commit to latest version * chore: clean up development branch * 1.1.0 Automatically generated by python-semantic-release * docs: update roadmap * 1.1.0 Automatically generated by python-semantic-release * chore: remove extra files created by semantic release * fix: rework semantic release * chore: add configuration to publishes artefacts to GitHub Releases * chore: remove auto created CHANGELOG.md * chore: release version 1.1.0 Automatically generated by python-semantic-release --------- Co-authored-by: github-actions <github-actions@github.com> * feat: add data model module to entirety * chore: smart data model crud * refactor: remove test code * chore: github link for data model loading * refactor: name change from github_link to schema_link * feat: add create service group from data model * feat: add boolean field to only apply required attrs from data model * chore: create temporary parser * feat: sdm parsed into entity form * fix: can't load schema from link because of wrong id of field * chore: add jsonschemaparser to requirements * chore: save local schema as temp file * chore: change sidebar icon * chore: update requirements.txt * feat: clean up functions for temporary files * chore: remove unused data model handler * chore: merge "create" and "creat from data model" in entity module * fix: deactivate load data model for updating * Feat: Load button added to devices moddule * Feat: Added Dropdown button to add new devices and batch device. * Feat: Added Dropdown button to add new devices and batch device. * Feat: Added URL for creating batch devices * Feat: Assign unique ID when new entity is loaded from datamodels * Feat: Assign unique ID when new entity is loaded from datamodels * Feat: Added parse_device function * chore: adjust views in devices modules * feat: parser for devices * chore: move and rename SmartDataModelEntitiesForm * feat: add load data model for devices * chore: add missing context * feat: implement data model load for servicegroups * fix: revise parse entity * chore: add initial resource for service group after loading from data model * fix: import error * chore: remove unused line * feat: implement viewer only for data model module * chore: rename edit button to inspect * build: included missing import * fix: save button not display for all users * chore: adjust layout for loading data models * chore: remove blank space in devices module * chore: update repo link for parser * fix: fixed search button displacement * chore: adjust layout of inspect button --------- Co-authored-by: JunsongDu <junsong.du@eonerc.rwth-aachen.de> Co-authored-by: Ganesh Thimmankatti <ganesh.thimmankatti@eonerc.rwth-aachen.de> * perf: update pillow version to include patch against DoS (#164) Co-authored-by: Sebastian Blechmann <51322874+SBlechmann@users.noreply.github.com> * chore: add load semantics option in .env.example * chore: implemented switch to load semantics module * chore: implemented switch for local auth or OIDC auth * build(entirety): change format of release notes (#172) * build: change format of release notes * docs (entirety): update commit convention in contribution guideline * fix(entirety): handle none scope --------- Co-authored-by: JunsongDu <101181614+djs0109@users.noreply.github.com> --------- Co-authored-by: Saira Bano <79838286+sbanoeon@users.noreply.github.com> Co-authored-by: github-actions <github-actions@github.com> Co-authored-by: Ganesh Thimmankatti <ganesh.thimmankatti@eonerc.rwth-aachen.de> Co-authored-by: Sebastian Blechmann <51322874+SBlechmann@users.noreply.github.com>
N5GEH · May 8, 2024 · 5a3f189 · 5a3f189
1 parent b7efea1
commit 5a3f189
Show file tree

Hide file tree

Showing 46 changed files with 1,362 additions and 295 deletions.
diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml
@@ -16,7 +16,7 @@ jobs:
         fetch-depth: 0
 
     - name: Python Semantic Release
-      uses: relekang/python-semantic-release@v8.0.8
+      uses: relekang/python-semantic-release@v9.4.2
       with:
         github_token: ${{ secrets.GITHUB_TOKEN  }}
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,12 @@
 exclude: '^static/[bootstrap|htmx]/'
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.5.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.1.1
     hooks:
     -   id: black
diff --git a/app/Entirety/.env.EXAMPLE b/app/Entirety/.env.EXAMPLE
@@ -2,6 +2,7 @@
 ENTITIES_LOAD=True
 DEVICES_LOAD=True
 NOTIFICATIONS_LOAD=True
+SEMANTICS_LOAD=True
 
 # Database
 DATABASE_PORT=5432
@@ -26,6 +27,7 @@ LOKI_PROTOCOL=http
 LOKI_SRC_HOST=entirety
 LOKI_TIMEZONE=Europe/Berlin
 
+LOCAL_AUTH=True
 # OIDC
 OIDC_OP_AUTHORIZATION_ENDPOINT=
 OIDC_OP_JWKS_ENDPOINT=

diff --git a/app/Entirety/devices/forms/servicegroups.py b/app/Entirety/devices/forms/servicegroups.py
@@ -1,5 +1,7 @@
 from crispy_forms.helper import FormHelper
 from django import forms
+from smartdatamodels.models import SmartDataModel
+import json
 
 
 class ServiceGroupBasic(forms.Form):
@@ -10,7 +12,7 @@ class ServiceGroupBasic(forms.Form):
             attrs={
                 "data-bs-toggle": "tooltip",
                 "title": "A string representing the southbound resource "
-                         "that will be used to provision a device, e.g. /iot/json",
+                "that will be used to provision a device, e.g. /iot/json",
             }
         ),
     )
@@ -35,10 +37,36 @@ class ServiceGroupBasic(forms.Form):
             }
         ),
     )
-    explicit_attrs = forms.BooleanField(label="Explicit Attributes", required=False, initial=False)
-    autoprovision = forms.BooleanField(label="Auto Provision", required=False, initial=True)
+    explicit_attrs = forms.BooleanField(
+        label="Explicit Attributes", required=False, initial=False
+    )
+    autoprovision = forms.BooleanField(
+        label="Auto Provision", required=False, initial=True
+    )
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.helper = FormHelper(self)
         self.helper.form_tag = False
+
+
+class SmartDataModelServicesForm(forms.Form):
+    select_data_model = forms.ChoiceField(label="Select a Data Model", required=True)
+    only_required_attrs = forms.BooleanField(
+        label="Only use required attributes", required=False, initial=False
+    )
+
+    def __init__(self, *args, **kwargs):
+        super(SmartDataModelServicesForm, self).__init__(*args, **kwargs)
+        qs = SmartDataModel.objects.all()
+        list_of_schemas = []
+        for set in qs:
+            list_of_schemas.append(
+                {"name": set.name, "value": json.dumps(set.jsonschema)}
+            )
+        list_of_schemas.append({"name": "..", "value": ".."})
+        self.fields["select_data_model"] = forms.ChoiceField(
+            choices=[(x.get("value"), x.get("name")) for x in list_of_schemas]
+        )
+        self.helper = FormHelper(self)
+        self.helper.form_tag = False
diff --git a/app/Entirety/devices/templates/devices/batch.html b/app/Entirety/devices/templates/devices/batch.html
@@ -3,25 +3,22 @@
 {% load compress %}
 {% load crispy_forms_tags %}
 
-
-{% block title %}Create Batch Devices {% endblock %}
+{% block title %}Create Batch Devices{% endblock %}
 
 {% block content %}
-    <span class="h3">Create Batch Devices </span>
+    <span class="h3">Create Batch Devices</span>
     <form method="post" action="" id="jsonForm" {% if form_submitted %}class="was-validated" {% endif %}>
         <div>
             {% csrf_token %}
             {% crispy json_form %}
             <button class="btn btn-primary rounded-pill" onclick="prettyJSON('id_device_json')" type="button">
                 <i class="bi bi-braces"></i> Beautify Json
             </button>
-            <button class="btn btn-warning rounded-pill" type="submit">Create</button>
+            <button class="btn btn-primary rounded-pill" type="submit">Create</button>
         </div>
     </form>
 
     {% compress js %}
         <script src="{% static 'js/json.js' %}"></script>
     {% endcompress %}
-
-
 {% endblock %}
diff --git a/app/Entirety/devices/templates/devices/datamodels.html b/app/Entirety/devices/templates/devices/datamodels.html
@@ -0,0 +1,29 @@
+{% extends '_base.html' %}
+{% load static %}
+{% load compress %}
+{% load crispy_forms_tags %}
+
+
+{% block title %}Create device or service group from data model {% endblock %}
+
+{% block content %}
+{#    <span class="h3">Create Device</span>#}
+    <form action="{% url 'projects:devices:create_submit_group_datamodel' project.uuid %}" method="post" novalidate>
+        {% crispy smart_data_model_form %}
+        <button class="btn btn-primary rounded-pill" type="submit">Confirm</button>
+    </form>
+{#    <form method="post" action="" id="jsonForm" {% if form_submitted %}class="was-validated" {% endif %}>#}
+{#        <div>#}
+{#            {% csrf_token %}#}
+{#            {{ json_form.media }}#}
+{#            {{ json_form }}#}
+{#            <button class="btn btn-warning rounded-pill" type="submit">Create</button>#}
+{#        </div>#}
+{#    </form>#}
+
+{#    {% compress js %}#}
+{#        <script src="{% static 'js/jsoneditor.js' %}"></script>#}
+{#    {% endcompress %}#}
+
+
+{% endblock %}
diff --git a/app/Entirety/devices/templates/devices/detail.html b/app/Entirety/devices/templates/devices/detail.html
@@ -1,4 +1,6 @@
 {% extends '_base.html' %}
+{% load crispy_forms_tags %}
+
 
 {% block title %}
     {% if action == "Create" %}
@@ -14,27 +16,31 @@
 
 {% block content %}
     {% if action == "Create" %}
-        <h3 class="mt-5">Create Device</h3>
+        <span class="h3">Create Device</span>
     {% elif action == "Edit" %}
-        <h3 class="mt-5">Edit Device</h3>
+        <span class="h3">Edit Device</span>
     {% elif action == "Create_Group" %}
-        <h3 class="mt-5">Create Service Group</h3>
+        <span class="h3">Create Service Group</span>
     {% elif action == "Edit_Group" %}
-        <h3 class="mt-5">Edit Service Group</h3>
+        <span class="h3">Edit Service Group</span>
     {% endif %}
 
     {% if action == "Create" %}
         <form action="{% url 'projects:devices:create_submit' project.uuid %}" method="post" novalidate>
+            {% crispy smart_data_model_form %}
+            <button class="btn btn-success bi bi-file-earmark-arrow-down rounded-pill" type="submit" name="load">Load</button>
     {% elif action == "Edit" %}
         <form action="{% url 'projects:devices:edit_submit' project.uuid %}" method="post" novalidate>
     {% elif action == "Create_Group" %}
         <form action="{% url 'projects:devices:create_submit_group' project.uuid %}" method="post" novalidate>
+            {% crispy smart_data_model_form %}
+            <button class="btn btn-success bi bi-file-earmark-arrow-down rounded-pill" type="submit" name="load">Load</button>
     {% elif action == "Edit_Group" %}
         <form action="{% url 'projects:devices:edit_submit_group' project.uuid %}" method="post" novalidate>
     {% endif %}
         {% csrf_token %}
             {% include 'accordion.html' %}
-            <button class="btn btn-primary rounded-pill" type="submit" {% if view_only %}
+            <button class="btn btn-primary rounded-pill" type="submit" name="submit" {% if view_only %}
                     disabled {% endif %}>Save</button>
         </form>
 

diff --git a/app/Entirety/devices/templates/devices/list.html b/app/Entirety/devices/templates/devices/list.html
@@ -52,29 +52,34 @@ <h3>Devices</h3>
          </div>
     </form>
 
-    <form action="{% url 'projects:devices:list_submit' project.uuid %}" method="POST" >
-        {% csrf_token %}
-        <div>
-            <button type="submit" class="btn btn-primary mb-1 ms-auto" data-toggle="tooltip" data-placement="bottom"
-                    title="Create Device" value="True" name="Create"  {% if view_only %}
-                    disabled {% endif %}>
+        <form action="{% url 'projects:devices:list_submit' project.uuid %}" method="POST">
+            {% csrf_token %}
+            <div>
+                <!-- Dropdown button for creating devices -->
+                <div class="btn-group">
+                    <button type="submit" class="btn btn-primary mb-1 ms-auto dropdown-toggle"
+                            data-bs-toggle="dropdown" aria-expanded="false" data-toggle="tooltip"
+                            data-placement="bottom" title="Create Device"value="True" name="Create"
+                            {% if view_only %} disabled {% endif %}>
                         <i class="bi bi-plus"></i>
-            </button>
-            <button type="submit" class="btn btn-primary mb-1 ms-auto" data-toggle="tooltip" data-placement="bottom"
-                    title="Create Multiple Device" value="True" name="BatchCreate"  {% if view_only %}
-                    disabled {% endif %}>
-                        <i class="bi bi-file-earmark-plus"></i>
-            </button>
-            <button type="button" class="btn btn-danger mb-1" data-toggle="tooltip" data-placement="bottom"
-                    title="Delete Device" data-bs-toggle="modal"  {% if not view_only %} data-bs-target="#deleteModal" {% endif %} {% if view_only %}
+                    </button>
+                    <ul class="dropdown-menu">
+                        <li><a class="dropdown-item" href="{% url 'projects:devices:create' project.uuid %}">Create Device</a></li>
+                        <li><a class="dropdown-item" href="{% url 'projects:devices:create_batch' project.uuid %}">Create
+                                Multiple Devices</a></li>
+                    </ul>
+                </div>
+
+                <button type="button" class="btn btn-danger mb-1" data-toggle="tooltip" data-placement="bottom"
+                        title="Delete Device" data-bs-toggle="modal" {% if not view_only %} data-bs-target="#deleteModal" {% endif %} {% if view_only %}
                     disabled {% endif %}>
-                        <i class="bi bi-trash"></i>
-            </button>
-            <button type="submit" class="btn btn-secondary mb-1" data-toggle="tooltip" data-placement="bottom"
-                    title="Inspect Device" value="True" name="Edit">
-                        <i class="bi bi-eye"></i>
-            </button>
-         </div>
+                    <i class="bi bi-trash"></i>
+                </button>
+                <button type="submit" class="btn btn-secondary mb-1" data-toggle="tooltip" data-placement="bottom"
+                        title="Inspect Device" value="True" name="Edit">
+                    <i class="bi bi-eye"></i>
+                </button>
+            </div>
 
             {% render_table tables.0 %}
 
@@ -169,24 +174,33 @@ <h3>Services Group</h3>
     </form>
 
     <form action="{% url 'projects:devices:list_submit_group' project.uuid %}" method="POST" >
-        {% csrf_token %}
-        <div>
-            <button type="submit" class="btn btn-primary mb-1 ms-auto" data-toggle="tooltip" data-placement="bottom"
-                    title="Create Service Group" value="True" name="Create_Group" {% if view_only %}
+            {% csrf_token %}
+            <div>
+                <!-- Dropdown button for creating service groups -->
+                <div class="btn-group">
+                    <button type="button" class="btn btn-primary mb-1 ms-auto dropdown-toggle"
+                            data-bs-toggle="dropdown" aria-expanded="false" data-toggle="tooltip"
+                            data-placement="bottom" title="Create Service Group"value="True" name="Create_Group" {% if view_only %}
                     disabled {% endif %}>
                         <i class="bi bi-plus"></i>
-            </button>
-            <button type="button" class="btn btn-danger mb-1" data-toggle="tooltip" data-placement="bottom"
-                    title="Delete Service Group" data-bs-toggle="modal"  {% if not view_only %}
+                    </button>
+                    <ul class="dropdown-menu">
+                        <li><a class="dropdown-item" href="{% url 'projects:devices:create_group' project.uuid %}">Create Service Group</a></li>
+{#                        <li><a class="dropdown-item" href="{% url 'projects:devices:create_batch_group' project.uuid %}">Create Multiple Service Groups</a></li>#}
+                    </ul>
+                </div>
+
+                <button type="button" class="btn btn-danger mb-1" data-toggle="tooltip" data-placement="bottom"
+                        title="Delete Service Group" data-bs-toggle="modal"  {% if not view_only %}
                     data-bs-target="#deleteModal_group" {% endif %} {% if view_only %}
                     disabled {% endif %}>
-                        <i class="bi bi-trash"></i>
-            </button>
-            <button type="submit" class="btn btn-secondary mb-1" data-toggle="tooltip" data-placement="bottom"
-                    title="Inspect Service Group" value="True" name="Edit_Group">
-                        <i class="bi bi-eye"></i>
-            </button>
-         </div>
+                    <i class="bi bi-trash"></i>
+                </button>
+                <button type="submit" class="btn btn-secondary mb-1" data-toggle="tooltip" data-placement="bottom"
+                        title="Inspect Service Group" value="True" name="Edit_Group">
+                    <i class="bi bi-eye"></i>
+                </button>
+            </div>
 
             {% render_table tables.1 %}
 

diff --git a/app/Entirety/devices/urls.py b/app/Entirety/devices/urls.py
@@ -7,21 +7,23 @@
     DeviceEditSubmitView,
     DeviceEditView,
     DeviceDeleteView,
-    DeviceBatchCreateView
+    DeviceCreateBatchView
 )
 from devices.views import (
     ServiceGroupListSubmitView,
     ServiceGroupCreateView,
     ServiceGroupCreateSubmitView,
     ServiceGroupEditView,
     ServiceGroupEditSubmitView,
-    ServiceGroupDeleteView
+    ServiceGroupDeleteView,
+    ServiceGroupDataModelCreateView,
+    ServiceGroupDataModelCreateSubmitView
 )
 
 app_name = "devices"
 urlpatterns = [
     path("create", DeviceCreateView.as_view(), name="create"),
-    path("batchcreate", DeviceBatchCreateView.as_view(), name="batchcreate"),
+    path("create/batch", DeviceCreateBatchView.as_view(), name="create_batch"),
     path("edit", DeviceEditView.as_view(), name="edit"),
     path("edit-submit", DeviceEditSubmitView.as_view(), name="edit_submit"),
     path("create-submit", DeviceCreateSubmitView.as_view(), name="create_submit"),
@@ -34,4 +36,7 @@
     path("create-submit-group", ServiceGroupCreateSubmitView.as_view(), name="create_submit_group"),
     path("list-submit-group", ServiceGroupListSubmitView.as_view(), name="list_submit_group"),
     path("delete-group", ServiceGroupDeleteView.as_view(), name="delete_group"),
+    path("create-group-datamodel", ServiceGroupDataModelCreateView.as_view(), name="create_group_datamodel"),
+    path("create-submit-group-datamodel", ServiceGroupDataModelCreateSubmitView.as_view(),
+         name="create_submit_group_datamodel")
 ]
diff --git a/app/Entirety/devices/utils.py b/app/Entirety/devices/utils.py
@@ -5,13 +5,23 @@
 from filip.clients.ngsi_v2 import IoTAClient
 from filip.models import FiwareHeader
 from filip.models.ngsi_v2.iot import Device, DeviceAttribute, DeviceCommand, ServiceGroup
-
+from filip.models.base import DataType
 
 # global settings
 prefix_attributes = "attributes"
 prefix_commands = "commands"
 
 
+JSONSchemaMap = {
+    "string": DataType.TEXT.value,
+    "number": DataType.NUMBER.value,
+    "integer": DataType.INTEGER.value,
+    "object": DataType.STRUCTUREDVALUE.value,
+    "array": DataType.ARRAY.value,
+    "boolean": DataType.BOOLEAN.value
+}
+
+
 def get_device_by_id(project: Project, device_id):
     """
     Get device by id for current project
@@ -379,3 +389,36 @@ def get_data_from_session(request, key):
         return request.session.pop(key)
     else:
         return None
+
+
+# TODO deprecate
+def _get_attributes_from_data_model(data_model, only_required_attrs):
+    properties = data_model.get("properties")
+    required_attrs = data_model.get("required")
+    print(f"required_attrs: {required_attrs}")
+    attributes = []
+    if properties:
+        for prop_name in properties:
+            if prop_name in ("id", "type"):
+                continue
+            if only_required_attrs and prop_name not in required_attrs:
+                continue
+            try:
+                attr_type = JSONSchemaMap[properties[prop_name]["type"]]
+            except:
+                attr_type = DataType.TEXT.value  # by default use text
+            attribute = {
+                "name": prop_name,
+                "type": attr_type,
+                "object_id": None
+            }
+            attributes.append(attribute)
+    return attributes
+
+
+
+# TODO deprecate
+def _get_entity_type_from_data_model(data_model):
+    if data_model.get("properties").get("type").get("enum"):
+        if len(data_model.get("properties").get("type").get("enum")) == 1:
+            return data_model.get("properties").get("type").get("enum")[0]