Merge pull request #101 from stat-kwon/master

Add state management fields to DataTable

src/spaceone/dashboard/manager/data_table_manager/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import ast
 import re
 from typing import Union, Literal, Tuple
 from jinja2 import Environment, meta
@@ -25,6 +26,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.df: Union[pd.DataFrame, None] = None
         self.jinja_variables = None
+        self.state = None
+        self.error_message = None
 
     def get_data_and_labels_info(self) -> Tuple[dict, dict]:
         raise NotImplementedError()
@@ -338,25 +341,34 @@ def change_global_variables(self, expression: str, vars: dict):
                     )
 
                 global_variable_value = vars[global_variable_key]
-
-                gv_type_map[global_variable_value] = type(global_variable_value)
+                gv_type = type(global_variable_value)
 
                 if isinstance(global_variable_value, int) or isinstance(
                     global_variable_value, float
                 ):
                     global_variable_value = str(global_variable_value)
 
+                if isinstance(global_variable_value, list):
+                    global_variable_value = str(global_variable_value)
+
+                gv_type_map[global_variable_value] = gv_type
+
                 expression = expression.replace(
                     global_variable_key, global_variable_value
                 )
 
         return expression, gv_type_map
 
     @staticmethod
-    def remove_jinja_braces(expression: str) -> str:
-        pattern = r"{{\s*(\w+)\s*}}"
-        new_expression = re.sub(pattern, r"\1", expression)
-        return new_expression
+    def remove_jinja_braces(expression: str) -> str | float | list:
+        if re.match(r"{{\s*(\w+)\s*}}", expression):
+            return re.sub(r"{{\s*(\w+)\s*}}", r"\1", expression)
+        elif re.match(r"{{\s*(\d+(\.\d+)?)\s*}}", expression):
+            result = re.sub(r"{{\s*(\d+(\.\d+)?)\s*}}", r"\1", expression)
+            return float(result)
+        else:
+            expression = expression.replace("{{", "").replace("}}", "").strip()
+            return ast.literal_eval(expression)
 
     @staticmethod
     def change_expression_data_type(expression: str, gv_type_map: dict) -> str:

src/spaceone/dashboard/manager/data_table_manager/data_source_manager.py

@@ -102,18 +102,23 @@ def load(
         end: str = None,
         vars: dict = None,
     ) -> pd.DataFrame:
-        start, end = self._get_time_from_granularity(granularity, start, end)
+        try:
+            start, end = self._get_time_from_granularity(granularity, start, end)
 
-        if self.timediff:
-            start, end = self._change_query_time(granularity, start, end)
+            if self.timediff:
+                start, end = self._change_query_time(granularity, start, end)
 
-        if self.source_type == "COST":
-            self._analyze_cost(granularity, start, end, vars)
-        elif self.source_type == "ASSET":
-            self._analyze_asset(granularity, start, end, vars)
+            if self.source_type == "COST":
+                self._analyze_cost(granularity, start, end, vars)
+            elif self.source_type == "ASSET":
+                self._analyze_asset(granularity, start, end, vars)
 
-        if additional_labels := self.options.get("additional_labels"):
-            self._add_labels(additional_labels)
+            if additional_labels := self.options.get("additional_labels"):
+                self._add_labels(additional_labels)
+        except Exception as e:
+            self.error_message = e.message if hasattr(e, "message") else str(e)
+            self.state = "UNAVAILABLE"
+            _LOGGER.error(f"[load] add {self.source_type} source error: {e}")
 
         return self.df
 
@@ -368,15 +373,19 @@ def _make_query(
         if self.filter:
             for filter_info in self.filter:
                 query_value = filter_info.get("v") or filter_info.get("value")
-
-                if isinstance(query_value, str) and self.is_jinja_expression(
-                    query_value
-                ):
+                if self.is_jinja_expression(query_value):
                     query_value, gv_type_map = self.change_global_variables(
                         query_value, vars
                     )
                     query_value = self.remove_jinja_braces(query_value)
-                    filter_info["v"] = [query_value]
+                    if (
+                        isinstance(query_value, str)
+                        or isinstance(query_value, int)
+                        or isinstance(query_value, float)
+                    ):
+                        filter_info["v"] = [query_value]
+                    elif isinstance(query_value, list):
+                        filter_info["v"] = query_value
 
         return {
             "granularity": granularity,

src/spaceone/dashboard/manager/data_table_manager/data_transformation_manager.py

@@ -80,16 +80,21 @@ def load(
         end: str = None,
         vars: dict = None,
     ) -> pd.DataFrame:
-        if self.operator == "JOIN":
-            self.join_data_tables(granularity, start, end, vars)
-        elif self.operator == "CONCAT":
-            self.concat_data_tables(granularity, start, end, vars)
-        elif self.operator == "AGGREGATE":
-            self.aggregate_data_table(granularity, start, end, vars)
-        elif self.operator == "QUERY":
-            self.query_data_table(granularity, start, end, vars)
-        elif self.operator == "EVAL":
-            self.evaluate_data_table(granularity, start, end, vars)
+        try:
+            if self.operator == "JOIN":
+                self.join_data_tables(granularity, start, end, vars)
+            elif self.operator == "CONCAT":
+                self.concat_data_tables(granularity, start, end, vars)
+            elif self.operator == "AGGREGATE":
+                self.aggregate_data_table(granularity, start, end, vars)
+            elif self.operator == "QUERY":
+                self.query_data_table(granularity, start, end, vars)
+            elif self.operator == "EVAL":
+                self.evaluate_data_table(granularity, start, end, vars)
+        except Exception as e:
+            self.error_message = e.message if hasattr(e, "message") else str(e)
+            self.state = "UNAVAILABLE"
+            _LOGGER.error(f"[load] {self.operator} operation error: {e}")
 
         return self.df
 

src/spaceone/dashboard/model/private_data_table/database.py

@@ -6,6 +6,10 @@
 class PrivateDataTable(MongoModel):
     data_table_id = StringField(max_length=40, generate_id="private-dt", unique=True)
     name = StringField(max_length=100, default=None, null=True)
+    state = StringField(
+        max_length=40, default="AVAILABLE", choices=("AVAILABLE", "UNAVAILABLE")
+    )
+    error_message = StringField(max_length=255, default=None, null=True)
     data_type = StringField(max_length=40, choices=("ADDED", "TRANSFORMED"))
     source_type = StringField(max_length=40, default=None, null=True)
     operator = StringField(max_length=40, default=None, null=True)
@@ -26,6 +30,8 @@ class PrivateDataTable(MongoModel):
     meta = {
         "updatable_fields": [
             "name",
+            "state",
+            "error_message",
             "options",
             "tags",
             "labels_info",
@@ -34,6 +40,7 @@ class PrivateDataTable(MongoModel):
         "minimal_fields": [
             "data_table_id",
             "name",
+            "state",
             "data_type",
             "source_type",
             "operator",
@@ -45,6 +52,7 @@ class PrivateDataTable(MongoModel):
         "ordering": ["name"],
         "indexes": [
             "name",
+            "state",
             "data_type",
             "source_type",
             "operator",

src/spaceone/dashboard/model/private_data_table/request.py

@@ -19,6 +19,7 @@ class PrivateDataTableAddRequest(BaseModel):
     name: Union[str, None] = None
     source_type: str
     options: dict
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     user_id: str
     domain_id: str
@@ -29,6 +30,7 @@ class PrivateDataTableTransformRequest(BaseModel):
     name: Union[str, None] = None
     operator: str
     options: dict
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     user_id: str
     domain_id: str
@@ -38,6 +40,7 @@ class PrivateDataTableUpdateRequest(BaseModel):
     data_table_id: str
     name: Union[str, None] = None
     options: Union[dict, None] = None
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     user_id: str
     domain_id: str
@@ -56,6 +59,7 @@ class PrivateDataTableLoadRequest(BaseModel):
     end: Union[str, None] = None
     sort: Union[list, None] = None
     page: Union[dict, None] = None
+    vars: Union[dict, None] = None
     user_id: str
     domain_id: str
 

src/spaceone/dashboard/model/private_data_table/response.py

@@ -9,13 +9,15 @@
 class PrivateDataTableResponse(BaseModel):
     data_table_id: Union[str, None] = None
     name: Union[str, None] = None
+    state: Union[str, None] = None
     data_type: Union[str, None] = None
     source_type: Union[str, None] = None
     operator: Union[str, None] = None
     options: Union[dict, None] = None
     tags: Union[dict, None] = None
     labels_info: Union[dict, None] = None
     data_info: Union[dict, None] = None
+    error_message: Union[str, None] = None
     dashboard_id: Union[str, None] = None
     widget_id: Union[str, None] = None
     user_id: Union[str, None] = None

src/spaceone/dashboard/model/public_data_table/database.py

@@ -6,6 +6,10 @@
 class PublicDataTable(MongoModel):
     data_table_id = StringField(max_length=40, generate_id="public-dt", unique=True)
     name = StringField(max_length=100, default=None, null=True)
+    state = StringField(
+        max_length=40, default="AVAILABLE", choices=("AVAILABLE", "UNAVAILABLE")
+    )
+    error_message = StringField(max_length=255, default=None, null=True)
     data_type = StringField(max_length=40, choices=("ADDED", "TRANSFORMED"))
     source_type = StringField(max_length=40, default=None, null=True)
     operator = StringField(max_length=40, default=None, null=True)
@@ -27,6 +31,8 @@ class PublicDataTable(MongoModel):
     meta = {
         "updatable_fields": [
             "name",
+            "state",
+            "error_message",
             "options",
             "tags",
             "labels_info",
@@ -37,6 +43,7 @@ class PublicDataTable(MongoModel):
         "minimal_fields": [
             "data_table_id",
             "name",
+            "state",
             "data_type",
             "source_type",
             "operator",
@@ -51,6 +58,7 @@ class PublicDataTable(MongoModel):
         "ordering": ["name"],
         "indexes": [
             "name",
+            "state",
             "data_type",
             "source_type",
             "operator",

src/spaceone/dashboard/model/public_data_table/request.py

@@ -19,6 +19,7 @@ class PublicDataTableAddRequest(BaseModel):
     name: Union[str, None] = None
     source_type: str
     options: dict
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     workspace_id: Union[str, None] = None
     domain_id: str
@@ -30,6 +31,7 @@ class PublicDataTableTransformRequest(BaseModel):
     name: Union[str, None] = None
     operator: str
     options: dict
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     workspace_id: Union[str, None] = None
     domain_id: str
@@ -40,6 +42,7 @@ class PublicDataTableUpdateRequest(BaseModel):
     data_table_id: str
     name: Union[str, None] = None
     options: Union[dict, None] = None
+    vars: Union[dict, None] = None
     tags: Union[dict, None] = None
     workspace_id: Union[str, None] = None
     domain_id: str
@@ -60,6 +63,7 @@ class PublicDataTableLoadRequest(BaseModel):
     end: Union[str, None] = None
     sort: Union[list, None] = None
     page: Union[dict, None] = None
+    vars: Union[dict, None] = None
     workspace_id: Union[str, list, None] = None
     domain_id: str
     user_projects: Union[list, None] = None

src/spaceone/dashboard/model/public_data_table/response.py

@@ -11,13 +11,15 @@
 class PublicDataTableResponse(BaseModel):
     data_table_id: Union[str, None] = None
     name: Union[str, None] = None
+    state: Union[str, None] = None
     data_type: Union[str, None] = None
     source_type: Union[str, None] = None
     operator: Union[str, None] = None
     options: Union[dict, None] = None
     tags: Union[dict, None] = None
     labels_info: Union[dict, None] = None
     data_info: Union[dict, None] = None
+    error_message: Union[str, None] = None
     dashboard_id: Union[str, None] = None
     widget_id: Union[str, None] = None
     resource_group: Union[ResourceGroup, None] = None