From f95c9c62e11d63238400156c4200603437526175 Mon Sep 17 00:00:00 2001 From: jolunoluo Date: Mon, 26 Feb 2024 21:36:55 +0800 Subject: [PATCH] (improvement)(headless) Parse sql variable --- .../headless/api/pojo/QueryParam.java | 4 +- ...asourceQuery.java => ModelDefineType.java} | 4 +- .../api/pojo/request/SqlExecuteReq.java | 8 ++- .../converter/CalculateAggConverter.java | 10 ++-- .../converter/ParserDefaultConverter.java | 12 ++--- .../converter/SqlVariableParseConverter.java | 49 +++++++++++++++++++ .../headless/core/pojo/MetricQueryParam.java | 4 +- .../headless/core/pojo/ViewQueryParam.java | 14 +----- .../headless/core/utils/ComponentFactory.java | 2 + .../core/utils/SqlVariableParseUtils.java | 18 ++++--- .../server/manager/ModelYamlManager.java | 4 +- .../server/rest/DatabaseController.java | 2 +- .../server/service/DatabaseService.java | 3 +- .../service/impl/DatabaseServiceImpl.java | 41 +++++++++------- 14 files changed, 111 insertions(+), 64 deletions(-) rename headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/{DatasourceQuery.java => ModelDefineType.java} (84%) create mode 100644 headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java index 8105a8ab7..26223b95d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java @@ -6,12 +6,11 @@ import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; +import lombok.Data; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import lombok.Data; @Data public class QueryParam { @@ -34,7 +33,6 @@ public class QueryParam { // metric private List metrics = new ArrayList(); private List dimensions; - private Map variables; private String where; private List order; private boolean nativeQuery = false; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DatasourceQuery.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java similarity index 84% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DatasourceQuery.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java index 4d7f4d7ba..1cc5b1081 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DatasourceQuery.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java @@ -5,7 +5,7 @@ * sql_query : view sql begin as select * table_query: dbName.tableName */ -public enum DatasourceQuery { +public enum ModelDefineType { SQL_QUERY("sql_query"), TABLE_QUERY("table_query"); @@ -13,7 +13,7 @@ public enum DatasourceQuery { private String name; - DatasourceQuery(String name) { + ModelDefineType(String name) { this.name = name; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java index 3ecb8b5b0..c61944c05 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java @@ -1,10 +1,12 @@ package com.tencent.supersonic.headless.api.pojo.request; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import com.tencent.supersonic.headless.api.pojo.SqlVariable; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.util.List; @Data public class SqlExecuteReq { @@ -16,6 +18,8 @@ public class SqlExecuteReq { @NotBlank(message = "sql can not be blank") private String sql; + private List sqlVariables; + public String getSql() { if (StringUtils.isNotBlank(sql) && sql.endsWith(";")) { sql = sql.substring(0, sql.length() - 1); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index 61cfbdaa4..757b7a82f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; /** * supplement the QueryStatement when query with custom aggregation method @@ -110,7 +111,6 @@ public void convert(QueryStatement queryStatement) throws Exception { EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); sqlCommend.setSql(viewQueryParam.getSql()); sqlCommend.setTables(viewQueryParam.getTables()); - sqlCommend.setVariables(viewQueryParam.getVariables()); sqlCommend.setSupportWith(viewQueryParam.isSupportWith()); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java index 641135ea1..8a77bf6a0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java @@ -1,20 +1,20 @@ package com.tencent.supersonic.headless.core.parser.converter; import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.headless.api.pojo.Param; import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + /** * HeadlessConverter default implement */ @@ -59,8 +59,6 @@ public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement metricQueryParam.setWhere(where); metricQueryParam.setOrder(queryParam.getOrders().stream() .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList())); - metricQueryParam.setVariables(queryParam.getParams().stream() - .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1))); metricQueryParam.setLimit(queryParam.getLimit()); // support detail query diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java new file mode 100644 index 000000000..fff8e79ac --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.headless.core.parser.converter; + +import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.Objects; + +@Slf4j +@Component("SqlVariableParseConverter") +public class SqlVariableParseConverter implements HeadlessConverter { + + @Override + public boolean accept(QueryStatement queryStatement) { + if (Objects.isNull(queryStatement.getQueryParam())) { + return false; + } + return true; + } + + @Override + public void convert(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + List modelResps = semanticSchemaResp.getModelResps(); + if (CollectionUtils.isEmpty(modelResps)) { + return; + } + for (ModelResp modelResp : modelResps) { + if (ModelDefineType.SQL_QUERY.getName() + .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) { + String sqlParsed = SqlVariableParseUtils.parse( + modelResp.getModelDetail().getSqlQuery(), + modelResp.getModelDetail().getSqlVariables(), + queryStatement.getQueryParam().getParams() + ); + DataSource dataSource = queryStatement.getSemanticModel() + .getDatasourceMap().get(modelResp.getBizName()); + dataSource.setSqlQuery(sqlParsed); + } + } + } +} \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java index c3caebf36..77b16f91d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java @@ -1,16 +1,14 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.common.pojo.ColumnOrder; -import java.util.List; -import java.util.Map; import lombok.Data; +import java.util.List; @Data public class MetricQueryParam { private List metrics; private List dimensions; - private Map variables; private String where; private Long limit; private List order; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java index 404699df2..97a258664 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java @@ -1,24 +1,14 @@ package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.headless.api.pojo.MetricTable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import lombok.Data; +import java.util.List; + @Data public class ViewQueryParam { - - private Map variables; private String sql = ""; private List tables; private boolean supportWith = true; private boolean withAlias = true; - - public Map getVariables() { - if (variables == null) { - variables = new HashMap<>(); - } - return variables; - } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index f94338c6c..1a55dbc53 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -3,6 +3,7 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.executor.JdbcExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter; import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer; import com.tencent.supersonic.headless.core.planner.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; @@ -83,6 +84,7 @@ private static void initQueryExecutors() { private static void initSemanticConverter() { headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); + headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class)); headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java index c1db3ac93..2a38bc383 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java @@ -30,13 +30,13 @@ public class SqlVariableParseUtils { private static final char delimiter = '$'; public static String parse(String sql, List sqlVariables, List params) { + Map variables = new HashMap<>(); if (CollectionUtils.isEmpty(sqlVariables)) { return sql; } - Map queryParams = new HashMap<>(); //1. handle default variable value sqlVariables.forEach(variable -> { - queryParams.put(variable.getName().trim(), + variables.put(variable.getName().trim(), getValues(variable.getValueType(), variable.getDefaultValues())); }); @@ -49,21 +49,25 @@ public static String parse(String sql, List sqlVariables, List list = map.get(p.getName()); if (!CollectionUtils.isEmpty(list)) { SqlVariable v = list.get(list.size() - 1); - queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue())); + variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue())); } } }); } - queryParams.forEach((k, v) -> { + variables.forEach((k, v) -> { if (v instanceof List && ((List) v).size() > 0) { v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString(); } - queryParams.put(k, v); + variables.put(k, v); }); + return parse(sql, variables); + } + + public static String parse(String sql, Map variables) { ST st = new ST(sql, delimiter, delimiter); - if (!CollectionUtils.isEmpty(queryParams)) { - queryParams.forEach(st::add); + if (!CollectionUtils.isEmpty(variables)) { + variables.forEach(st::add); } return st.render(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index b73a758ac..333f608d4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -4,7 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery; +import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; @@ -46,7 +46,7 @@ public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, .collect(Collectors.toList())); dataModelYamlTpl.setName(modelResp.getBizName()); dataModelYamlTpl.setSourceId(modelResp.getDatabaseId()); - if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) { + if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) { dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery()); } else { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 8e3211357..5470e83d5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -73,7 +73,7 @@ public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return databaseService.executeSql(sqlExecuteReq.getSql(), sqlExecuteReq.getId(), user); + return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user); } @RequestMapping("/getDbNames/{id}") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index 99a1f3d97..af0fe3a9d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -2,6 +2,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; +import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; @@ -13,7 +14,7 @@ public interface DatabaseService { SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp); - SemanticQueryResp executeSql(String sql, Long id, User user); + SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user); DatabaseResp getDatabase(Long id, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 1854434d8..d75e8034e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.server.service.impl; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq; +import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; @@ -11,6 +12,7 @@ import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; import com.tencent.supersonic.headless.core.utils.SqlUtils; +import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; @@ -116,32 +118,19 @@ public DatabaseResp getDatabase(Long id) { @Override public DatabaseResp getDatabase(Long id, User user) { DatabaseResp databaseResp = getDatabase(id); - if (!databaseResp.getAdmins().contains(user.getName()) - && !databaseResp.getViewers().contains(user.getName()) - && !databaseResp.getCreatedBy().equals(user.getName())) { - throw new InvalidPermissionException("您暂无查看该数据库详情的权限, 请联系创建人: " - + databaseResp.getCreatedBy()); - } + checkPermission(databaseResp, user); return databaseResp; } @Override - public SemanticQueryResp executeSql(String sql, Long id, User user) { + public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user) { DatabaseResp databaseResp = getDatabase(id); if (databaseResp == null) { return new SemanticQueryResp(); } - List admins = databaseResp.getAdmins(); - List viewers = databaseResp.getViewers(); - if (!admins.contains(user.getName()) - && !viewers.contains(user.getName()) - && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName()) - && !user.isSuperAdmin()) { - String message = String.format("您暂无当前数据库%s权限, 请联系数据库管理员%s开通", - databaseResp.getName(), - String.join(",", admins)); - throw new RuntimeException(message); - } + checkPermission(databaseResp, user); + String sql = sqlExecuteReq.getSql(); + sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList()); return executeSql(sql, databaseResp); } @@ -195,4 +184,18 @@ public SemanticQueryResp getColumns(Long id, String db, String table) { return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp)); } + private void checkPermission(DatabaseResp databaseResp, User user) { + List admins = databaseResp.getAdmins(); + List viewers = databaseResp.getViewers(); + if (!admins.contains(user.getName()) + && !viewers.contains(user.getName()) + && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName()) + && !user.isSuperAdmin()) { + String message = String.format("您暂无当前数据库%s权限, 请联系数据库创建人:%s开通", + databaseResp.getName(), + databaseResp.getCreatedBy()); + throw new RuntimeException(message); + } + } + }