(improvement)(headless) Parse sql variable

tencentmusic · Feb 26, 2024 · f95c9c6 · f95c9c6
1 parent 0beb3ce
commit f95c9c6
Show file tree

Hide file tree

Showing 14 changed files with 111 additions and 64 deletions.
diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java
@@ -6,12 +6,11 @@
 import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
+import lombok.Data;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import lombok.Data;
 
 @Data
 public class QueryParam {
@@ -34,7 +33,6 @@ public class QueryParam {
     // metric
     private List<String> metrics = new ArrayList();
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private List<ColumnOrder> order;
     private boolean nativeQuery = false;

diff --git a/...dless/api/pojo/enums/DatasourceQuery.java → ...dless/api/pojo/enums/ModelDefineType.java b/...dless/api/pojo/enums/DatasourceQuery.java → ...dless/api/pojo/enums/ModelDefineType.java
@@ -5,15 +5,15 @@
  * sql_query : view sql begin as select
  * table_query: dbName.tableName
  */
-public enum DatasourceQuery {
+public enum ModelDefineType {
 
     SQL_QUERY("sql_query"),
     TABLE_QUERY("table_query");
 
     private String name;
 
 
-    DatasourceQuery(String name) {
+    ModelDefineType(String name) {
         this.name = name;
     }
 

diff --git a/...ess/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java b/...ess/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java
@@ -1,10 +1,12 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
 
-import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
+import com.tencent.supersonic.headless.api.pojo.SqlVariable;
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.util.List;
 
 @Data
 public class SqlExecuteReq {
@@ -16,6 +18,8 @@ public class SqlExecuteReq {
     @NotBlank(message = "sql can not be blank")
     private String sql;
 
+    private List<SqlVariable> sqlVariables;
+
     public String getSql() {
         if (StringUtils.isNotBlank(sql) && sql.endsWith(";")) {
             sql = sql.substring(0, sql.length() - 1);

diff --git a/...ain/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/...ain/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
@@ -9,18 +9,19 @@
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 /**
  * supplement the QueryStatement when query with custom aggregation method
@@ -110,7 +111,6 @@ public void convert(QueryStatement queryStatement) throws Exception {
                 EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
         sqlCommend.setSql(viewQueryParam.getSql());
         sqlCommend.setTables(viewQueryParam.getTables());
-        sqlCommend.setVariables(viewQueryParam.getVariables());
         sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
     }
 

diff --git a/...in/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/...in/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
@@ -1,20 +1,20 @@
 package com.tencent.supersonic.headless.core.parser.converter;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import com.tencent.supersonic.headless.api.pojo.Param;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
-import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 /**
  * HeadlessConverter default implement
  */
@@ -59,8 +59,6 @@ public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement
         metricQueryParam.setWhere(where);
         metricQueryParam.setOrder(queryParam.getOrders().stream()
                 .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
-        metricQueryParam.setVariables(queryParam.getParams().stream()
-                .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
         metricQueryParam.setLimit(queryParam.getLimit());
 
         // support detail query

diff --git a/...java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java b/...java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java
@@ -0,0 +1,49 @@
+package com.tencent.supersonic.headless.core.parser.converter;
+
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
+import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+@Slf4j
+@Component("SqlVariableParseConverter")
+public class SqlVariableParseConverter implements HeadlessConverter {
+
+    @Override
+    public boolean accept(QueryStatement queryStatement) {
+        if (Objects.isNull(queryStatement.getQueryParam())) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void convert(QueryStatement queryStatement) {
+        SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
+        List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
+        if (CollectionUtils.isEmpty(modelResps)) {
+            return;
+        }
+        for (ModelResp modelResp : modelResps) {
+            if (ModelDefineType.SQL_QUERY.getName()
+                    .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) {
+                String sqlParsed = SqlVariableParseUtils.parse(
+                        modelResp.getModelDetail().getSqlQuery(),
+                        modelResp.getModelDetail().getSqlVariables(),
+                        queryStatement.getQueryParam().getParams()
+                );
+                DataSource dataSource = queryStatement.getSemanticModel()
+                        .getDatasourceMap().get(modelResp.getBizName());
+                dataSource.setSqlQuery(sqlParsed);
+            }
+        }
+    }
+}
diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java
@@ -1,16 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
+import java.util.List;
 
 @Data
 public class MetricQueryParam {
 
     private List<String> metrics;
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private Long limit;
     private List<ColumnOrder> order;

diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java
@@ -1,24 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ViewQueryParam {
-
-    private Map<String, String> variables;
     private String sql = "";
     private List<MetricTable> tables;
     private boolean supportWith = true;
     private boolean withAlias = true;
-
-    public Map<String, String> getVariables() {
-        if (variables == null) {
-            variables = new HashMap<>();
-        }
-        return variables;
-    }
 }
diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java
@@ -3,6 +3,7 @@
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
+import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter;
 import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
 import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
@@ -83,6 +84,7 @@ private static void initQueryExecutors() {
 
     private static void initSemanticConverter() {
         headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
+        headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class));
         headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
         headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
     }

diff --git a/.../core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java b/.../core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java
@@ -30,13 +30,13 @@ public class SqlVariableParseUtils {
     private static final char delimiter = '$';
 
     public static String parse(String sql, List<SqlVariable> sqlVariables, List<Param> params) {
+        Map<String, Object> variables = new HashMap<>();
         if (CollectionUtils.isEmpty(sqlVariables)) {
             return sql;
         }
-        Map<String, Object> queryParams = new HashMap<>();
         //1. handle default variable value
         sqlVariables.forEach(variable -> {
-            queryParams.put(variable.getName().trim(),
+            variables.put(variable.getName().trim(),
                     getValues(variable.getValueType(), variable.getDefaultValues()));
         });
 
@@ -49,21 +49,25 @@ public static String parse(String sql, List<SqlVariable> sqlVariables, List<Para
                     List<SqlVariable> list = map.get(p.getName());
                     if (!CollectionUtils.isEmpty(list)) {
                         SqlVariable v = list.get(list.size() - 1);
-                        queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
+                        variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
                     }
                 }
             });
         }
 
-        queryParams.forEach((k, v) -> {
+        variables.forEach((k, v) -> {
             if (v instanceof List && ((List) v).size() > 0) {
                 v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
             }
-            queryParams.put(k, v);
+            variables.put(k, v);
         });
+        return parse(sql, variables);
+    }
+
+    public static String parse(String sql, Map<String, Object> variables) {
         ST st = new ST(sql, delimiter, delimiter);
-        if (!CollectionUtils.isEmpty(queryParams)) {
-            queryParams.forEach(st::add);
+        if (!CollectionUtils.isEmpty(variables)) {
+            variables.forEach(st::add);
         }
         return st.render();
     }

diff --git a/...server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/...server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
@@ -4,7 +4,7 @@
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
-import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery;
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
@@ -46,7 +46,7 @@ public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
                 .collect(Collectors.toList()));
         dataModelYamlTpl.setName(modelResp.getBizName());
         dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
-        if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
+        if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {
             dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
         } else {
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());

diff --git a/.../server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/.../server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
@@ -73,7 +73,7 @@ public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
             HttpServletRequest request,
             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return databaseService.executeSql(sqlExecuteReq.getSql(), sqlExecuteReq.getId(), user);
+        return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
     }
 
     @RequestMapping("/getDbNames/{id}")

diff --git a/.../server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/.../server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
@@ -2,6 +2,7 @@
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
+import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
@@ -13,7 +14,7 @@ public interface DatabaseService {
 
     SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp);
 
-    SemanticQueryResp executeSql(String sql, Long id, User user);
+    SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user);
 
     DatabaseResp getDatabase(Long id, User user);