From 04aeab0cacf11df43497c6cb86dc52b54905d703 Mon Sep 17 00:00:00 2001 From: EEA Jenkins <2368628+eea-jenkins@users.noreply.github.com> Date: Mon, 13 May 2024 16:33:01 +0300 Subject: [PATCH 01/14] Back to devel --- docs/HISTORY.txt | 3 +++ eea/api/dataconnector/version.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 3c2a755..b3540c3 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,6 +1,9 @@ Changelog ========= +10.3-dev0 - (unreleased) +--------------------------- + 10.2 - (2024-05-08) --------------------------- * Change: Release diff --git a/eea/api/dataconnector/version.txt b/eea/api/dataconnector/version.txt index e2498ea..fb54e4b 100644 --- a/eea/api/dataconnector/version.txt +++ b/eea/api/dataconnector/version.txt @@ -1 +1 @@ -10.2 +10.3-dev0 From 0614357783cea49c86fb6c32005281ca8dabf85b Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 14:17:38 +0300 Subject: [PATCH 02/14] Create pagination for files --- eea/api/dataconnector/behavior.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 3ded1fa..4ba52b9 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -21,8 +21,8 @@ from .interfaces import IFileDataProvider from .interfaces import IElasticConnector from .interfaces import IFigureNote - - +from eea.api.dataconnector.queryparser import parseQuery +from plone.restapi.deserializer import json_body logger = logging.getLogger(__name__) @@ -58,9 +58,12 @@ def provided_data(self): """provided data""" field = IPrimaryFieldInfo(self.context) + page = json_body(self.request).get("form",{}).get("p", 0) + nrOfHits = json_body(self.request).get("form",{}).get("nrOfHits", 0) + if not field.value: return [] - + text = field.value.data f = StringIO(text.decode("utf-8-sig")) try: @@ -75,11 +78,22 @@ def provided_data(self): keys = rows[0] data = [] - - for index, row in enumerate(rows[1:]): - data.append({}) - for (i, k) in enumerate(keys): - data[index][k] = row[i] + cnt = 0; + if page>= 1 and nrOfHits >= 1: + for index, row in enumerate(rows[1:]): + if index+1 > (page-1)*nrOfHits and index+1 <= page*nrOfHits: + data.append({}) + for (i, k) in enumerate(keys): + data[cnt][k] = row[i] + cnt = cnt+1; + + + else: + for index, row in enumerate(rows[1:]): + data.append({}) + for (i, k) in enumerate(keys): + data[index][k] = row[i] + data_query = computeDataQuery(self.request) From 12a665a970b8f5173f60e23541d6056e20dcbffc Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 21:51:39 +0300 Subject: [PATCH 03/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 4ba52b9..e88fe0d 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -21,7 +21,6 @@ from .interfaces import IFileDataProvider from .interfaces import IElasticConnector from .interfaces import IFigureNote -from eea.api.dataconnector.queryparser import parseQuery from plone.restapi.deserializer import json_body logger = logging.getLogger(__name__) @@ -78,22 +77,11 @@ def provided_data(self): keys = rows[0] data = [] - cnt = 0; - if page>= 1 and nrOfHits >= 1: - for index, row in enumerate(rows[1:]): - if index+1 > (page-1)*nrOfHits and index+1 <= page*nrOfHits: - data.append({}) - for (i, k) in enumerate(keys): - data[cnt][k] = row[i] - cnt = cnt+1; - - - else: - for index, row in enumerate(rows[1:]): + + for index, row in enumerate(rows[((page-1)*nrOfHits+1):(page*nrOfHits+1)] if page>= 1 and nrOfHits >= 1 else rows[1:]): data.append({}) for (i, k) in enumerate(keys): data[index][k] = row[i] - data_query = computeDataQuery(self.request) From 243aada57cc6300bc3711198758be8a04f8daa8b Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 21:53:59 +0300 Subject: [PATCH 04/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index e88fe0d..323441f 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -1,4 +1,5 @@ """ behavior module """ + import csv import logging from io import StringIO @@ -22,6 +23,7 @@ from .interfaces import IElasticConnector from .interfaces import IFigureNote from plone.restapi.deserializer import json_body + logger = logging.getLogger(__name__) @@ -36,9 +38,7 @@ class DataConnector(MetadataBase): endpoint_url = DCFieldProperty(IDataConnector["endpoint_url"]) sql_query = DCFieldProperty(IDataConnector["sql_query"]) parameters = DCFieldProperty(IDataConnector["parameters"]) - required_parameters = DCFieldProperty( - IDataConnector["required_parameters"] - ) + required_parameters = DCFieldProperty(IDataConnector["required_parameters"]) collate = DCFieldProperty(IDataConnector["collate"]) readme = DCFieldProperty(IDataConnector["readme"]) @@ -57,12 +57,12 @@ def provided_data(self): """provided data""" field = IPrimaryFieldInfo(self.context) - page = json_body(self.request).get("form",{}).get("p", 0) - nrOfHits = json_body(self.request).get("form",{}).get("nrOfHits", 0) + page = json_body(self.request).get("form", {}).get("p", 0) + nrOfHits = json_body(self.request).get("form", {}).get("nrOfHits", 0) if not field.value: return [] - + text = field.value.data f = StringIO(text.decode("utf-8-sig")) try: @@ -78,10 +78,14 @@ def provided_data(self): keys = rows[0] data = [] - for index, row in enumerate(rows[((page-1)*nrOfHits+1):(page*nrOfHits+1)] if page>= 1 and nrOfHits >= 1 else rows[1:]): - data.append({}) - for (i, k) in enumerate(keys): - data[index][k] = row[i] + for index, row in enumerate( + rows[((page - 1) * nrOfHits + 1) : (page * nrOfHits + 1)] + if page >= 1 and nrOfHits >= 1 + else rows[1:] + ): + data.append({}) + for i, k in enumerate(keys): + data[index][k] = row[i] data_query = computeDataQuery(self.request) @@ -104,9 +108,9 @@ def __init__(self, context, request): def provided_data(self): """provided data""" - widget = getattr(self.context, 'elastic_csv_widget', None) + widget = getattr(self.context, "elastic_csv_widget", None) - data = widget['tableData'] if widget else {} + data = widget["tableData"] if widget else {} return { "results": data, @@ -151,14 +155,10 @@ class ConnectorDataParameters(MetadataBase): class ElasticConnectorWidget(MetadataBase): """Build csv data from ES data""" - elastic_csv_widget = DCFieldProperty( - IElasticConnector["elastic_csv_widget"] - ) + elastic_csv_widget = DCFieldProperty(IElasticConnector["elastic_csv_widget"]) class FigureNoteField(MetadataBase): """Insert Figure Note field""" - figure_note = DCFieldProperty( - IFigureNote["figure_note"] - ) + figure_note = DCFieldProperty(IFigureNote["figure_note"]) From 82b65a4812e12b4b7928156066c7d5c83768da56 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:00:01 +0300 Subject: [PATCH 05/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 323441f..b044a4e 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -38,7 +38,8 @@ class DataConnector(MetadataBase): endpoint_url = DCFieldProperty(IDataConnector["endpoint_url"]) sql_query = DCFieldProperty(IDataConnector["sql_query"]) parameters = DCFieldProperty(IDataConnector["parameters"]) - required_parameters = DCFieldProperty(IDataConnector["required_parameters"]) + required_parameters = DCFieldProperty( + IDataConnector["required_parameters"]) collate = DCFieldProperty(IDataConnector["collate"]) readme = DCFieldProperty(IDataConnector["readme"]) @@ -155,7 +156,8 @@ class ConnectorDataParameters(MetadataBase): class ElasticConnectorWidget(MetadataBase): """Build csv data from ES data""" - elastic_csv_widget = DCFieldProperty(IElasticConnector["elastic_csv_widget"]) + elastic_csv_widget = DCFieldProperty( + IElasticConnector["elastic_csv_widget"]) class FigureNoteField(MetadataBase): From 59cd4240f12fc1f7dc33695dc5b45f9435124ea0 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:05:50 +0300 Subject: [PATCH 06/14] Fix pylint --- eea/api/dataconnector/behavior.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index b044a4e..e25a767 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -80,7 +80,7 @@ def provided_data(self): data = [] for index, row in enumerate( - rows[((page - 1) * nrOfHits + 1) : (page * nrOfHits + 1)] + rows[((page - 1) * nrOfHits + 1):(page * nrOfHits + 1)] if page >= 1 and nrOfHits >= 1 else rows[1:] ): From 3a530aca90f17485297cf52950f9303514ddbf08 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:12:55 +0300 Subject: [PATCH 07/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index e25a767..7c4ff17 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -12,6 +12,7 @@ from zope.publisher.interfaces.browser import IBrowserRequest from eea.api.dataconnector.queryparser import computeDataQuery from eea.api.dataconnector.queryfilter import filteredData +from plone.restapi.deserializer import json_body from .interfaces import IConnectorDataParameters from .interfaces import IDataConnector from .interfaces import IDataProvider @@ -22,7 +23,7 @@ from .interfaces import IFileDataProvider from .interfaces import IElasticConnector from .interfaces import IFigureNote -from plone.restapi.deserializer import json_body + logger = logging.getLogger(__name__) From 67cec1cce560edcca5092396980e26100548396f Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:19:01 +0300 Subject: [PATCH 08/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 7c4ff17..b521d92 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -3,26 +3,23 @@ import csv import logging from io import StringIO -from plone.app.dexterity.behaviors.metadata import DCFieldProperty -from plone.app.dexterity.behaviors.metadata import MetadataBase + +from plone.app.dexterity.behaviors.metadata import DCFieldProperty, MetadataBase from plone.dexterity.interfaces import IDexterityContent from plone.rfc822.interfaces import IPrimaryFieldInfo +from plone.restapi.deserializer import json_body from zope.component import adapter from zope.interface import implementer from zope.publisher.interfaces.browser import IBrowserRequest + from eea.api.dataconnector.queryparser import computeDataQuery from eea.api.dataconnector.queryfilter import filteredData -from plone.restapi.deserializer import json_body -from .interfaces import IConnectorDataParameters -from .interfaces import IDataConnector -from .interfaces import IDataProvider -from .interfaces import IDataVisualization -from .interfaces import IMaps -from .interfaces import IMapVisualization -from .interfaces import ITableauVisualization -from .interfaces import IFileDataProvider -from .interfaces import IElasticConnector -from .interfaces import IFigureNote + +from .interfaces import ( + IConnectorDataParameters, IDataConnector, IDataProvider, IDataVisualization, + IMaps, IMapVisualization, ITableauVisualization, IFileDataProvider, + IElasticConnector, IFigureNote +) logger = logging.getLogger(__name__) From 917fd5c82bba603e990947feecf278a4f6c824d3 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:23:16 +0300 Subject: [PATCH 09/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index b521d92..05b30c4 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -4,7 +4,10 @@ import logging from io import StringIO -from plone.app.dexterity.behaviors.metadata import DCFieldProperty, MetadataBase +from plone.app.dexterity.behaviors.metadata import ( + DCFieldProperty, + MetadataBase +) from plone.dexterity.interfaces import IDexterityContent from plone.rfc822.interfaces import IPrimaryFieldInfo from plone.restapi.deserializer import json_body @@ -16,8 +19,11 @@ from eea.api.dataconnector.queryfilter import filteredData from .interfaces import ( - IConnectorDataParameters, IDataConnector, IDataProvider, IDataVisualization, - IMaps, IMapVisualization, ITableauVisualization, IFileDataProvider, + IConnectorDataParameters, IDataConnector, + IDataProvider, + IDataVisualization, + IMaps, IMapVisualization, ITableauVisualization, + IFileDataProvider, IElasticConnector, IFigureNote ) From 9b97aa34629d91c9876afcb494cae856e34f2240 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:25:03 +0300 Subject: [PATCH 10/14] fix pylint --- eea/api/dataconnector/behavior.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 05b30c4..88334af 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -19,10 +19,10 @@ from eea.api.dataconnector.queryfilter import filteredData from .interfaces import ( - IConnectorDataParameters, IDataConnector, - IDataProvider, + IConnectorDataParameters, IDataConnector, + IDataProvider, IDataVisualization, - IMaps, IMapVisualization, ITableauVisualization, + IMaps, IMapVisualization, ITableauVisualization, IFileDataProvider, IElasticConnector, IFigureNote ) From 2cabce3f6f05e7059bef9123c048880eecb9f554 Mon Sep 17 00:00:00 2001 From: dobri1408 <50819975+dobri1408@users.noreply.github.com> Date: Mon, 20 May 2024 22:26:49 +0300 Subject: [PATCH 11/14] Update behavior.py --- eea/api/dataconnector/behavior.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eea/api/dataconnector/behavior.py b/eea/api/dataconnector/behavior.py index 88334af..6a4fee7 100644 --- a/eea/api/dataconnector/behavior.py +++ b/eea/api/dataconnector/behavior.py @@ -5,7 +5,7 @@ from io import StringIO from plone.app.dexterity.behaviors.metadata import ( - DCFieldProperty, + DCFieldProperty, MetadataBase ) from plone.dexterity.interfaces import IDexterityContent From b5a5039e13a3b62690db879a2d612074a8079b64 Mon Sep 17 00:00:00 2001 From: nileshgulia1 Date: Tue, 11 Jun 2024 18:10:02 +0530 Subject: [PATCH 12/14] fix: get visualization @id from its metadata Content --- eea/api/dataconnector/browser/blocks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eea/api/dataconnector/browser/blocks.py b/eea/api/dataconnector/browser/blocks.py index 111e992..dc63a26 100644 --- a/eea/api/dataconnector/browser/blocks.py +++ b/eea/api/dataconnector/browser/blocks.py @@ -266,9 +266,9 @@ def __call__(self, value): return { **value, "vis_url": vis_url, "visualization": { - **getMetadata(doc_serializer), **getVisualization(serializer=doc_serializer, layout=use_data_sources), + **getMetadata(doc_serializer), } } return {**value, "vis_url": uid_to_url(value.get('vis_url'))} @@ -369,8 +369,8 @@ def __call__(self, value): return { **value, "tableau_vis_url": tableau_vis_url, "tableau_visualization": { - **getMetadata(doc_serializer), **doc_serializer.get('tableau_visualization', {}), + **getMetadata(doc_serializer), } } return { @@ -474,8 +474,8 @@ def __call__(self, value): return { **value, "vis_url": vis_url, "map_visualization_data": { - **getMetadata(doc_serializer), **doc_serializer.get('map_visualization_data', {}), + **getMetadata(doc_serializer), } } return { @@ -574,8 +574,8 @@ def __call__(self, value): if doc_serializer: return { **value, "maps": { - **getMetadata(doc_serializer), **doc_serializer.get('maps', {}), + **getMetadata(doc_serializer), } } return value From 624a98a75de0bebe2a17777effa3bc64c3208b06 Mon Sep 17 00:00:00 2001 From: EEA Jenkins Date: Tue, 11 Jun 2024 17:26:17 +0300 Subject: [PATCH 13/14] Updated version to 10.3 --- eea/api/dataconnector/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eea/api/dataconnector/version.txt b/eea/api/dataconnector/version.txt index fb54e4b..260e375 100644 --- a/eea/api/dataconnector/version.txt +++ b/eea/api/dataconnector/version.txt @@ -1 +1 @@ -10.3-dev0 +10.3 From 08ff432c506f86f8a588dd6a0021b56aa2ca427e Mon Sep 17 00:00:00 2001 From: EEA Jenkins Date: Tue, 11 Jun 2024 17:26:21 +0300 Subject: [PATCH 14/14] Updated changelog - removed develop information --- docs/HISTORY.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index b3540c3..8b14ded 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,8 +1,10 @@ Changelog ========= -10.3-dev0 - (unreleased) +10.3 - (2024-06-11) --------------------------- +* Change: Release + [claudiaifrim] 10.2 - (2024-05-08) ---------------------------