diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index e72d58f66143..65ef1a37ab32 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -108,25 +108,41 @@ protected Skip skipIndex(ValueType valueType, boolean hasOptionSet) { return skipIndex ? Skip.SKIP : Skip.INCLUDE; } - protected String getSelectClause(ValueType valueType, String columnName) { - return getSelectClauseInternal(valueType, columnName, false); + /** + * Returns a select expression for a data element value, handling casting to the appropriate data + * type based on the given value type. + * + * @param valueType the {@link ValueType}. + * @param columnName the column name. + * @return a select expression. + */ + protected String getSelectExpression(ValueType valueType, String columnName) { + return getSelectExpressionInternal(valueType, columnName, false); } - protected String getSelectClauseForTea(ValueType valueType, String columnName) { - return getSelectClauseInternal(valueType, columnName, true); + /** + * Returns a select expression for a tracked entity attribute, handling casting to the appropriate + * data type based on the given value type. + * + * @param valueType the {@link ValueType}. + * @param columnName the column name. + * @return a select expression. + */ + protected String getSelectExpressionForAttribute(ValueType valueType, String columnName) { + return getSelectExpressionInternal(valueType, columnName, true); } /** - * Returns the select clause, potentially with a cast statement, based on the given value type. - * This internal method handles both Data Value and Tracked Entity Attribute (TEA) select clauses. + * Returns a select expression, potentially with a cast statement, based on the given value type. + * Handles data element and tracked entity attribute select expressions. * - * @param valueType The value type to represent as database column type - * @param columnName The name of the column to be selected - * @param isTeaContext Whether the selection is in the context of a Tracked Entity Attribute. When - * true, organization unit selections will include an additional subquery wrapper - * @return A SQL select expression appropriate for the given value type and context + * @param valueType the {@link ValueType} to represent as database column type. + * @param columnName the name of the column to be selected. + * @param isTeaContext whether the selection is in the context of a tracked entity attribute. When + * true, organization unit selections will include an additional subquery wrapper. + * @return A SQL select expression appropriate for the given value type and context. */ - private String getSelectClauseInternal( + private String getSelectExpressionInternal( ValueType valueType, String columnName, boolean isTeaContext) { String doubleType = sqlBuilder.dataTypeDouble(); @@ -201,7 +217,7 @@ protected List getTrackedEntityAttributeColumns(Program pr attribute.isNumericType() ? getNumericClause() : attribute.isDateType() ? getDateClause() : ""; - String select = getSelectClauseForTea(attribute.getValueType(), "value"); + String select = getSelectExpressionForAttribute(attribute.getValueType(), "value"); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); String sql = @@ -253,18 +269,18 @@ protected List getTrackedEntityAttributeColumns(Program pr * * @param attribute the {@link TrackedEntityAttribute}. * @param fromType the sql snippet related to "from" part - * @param dataClause the data type related clause like "NUMERIC" - * @return + * @param dataClause the data type related clause like "NUMERIC". + * @return a select statement. */ protected String selectForInsert( TrackedEntityAttribute attribute, String fromType, String dataClause) { return replaceQualify( """ - (select ${fromType} from ${trackedentityattributevalue} \ - where trackedentityid=en.trackedentityid \ - and trackedentityattributeid=${attributeId}\ - ${dataClause})\ - ${closingParentheses} as ${attributeUid}""", + (select ${fromType} from ${trackedentityattributevalue} \ + where trackedentityid=en.trackedentityid \ + and trackedentityattributeid=${attributeId}\ + ${dataClause})\ + ${closingParentheses} as ${attributeUid}""", Map.of( "fromType", fromType, "dataClause", dataClause, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index d9d27b66892d..2a4b27ff8c8b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -746,11 +746,11 @@ protected AnalyticsTableColumn getPartitionColumn() { // is part of the previous PR (https://github.com/dhis2/dhis2-core/pull/19131/files) .selectExpression( """ - CASE - WHEN ev.status = 'SCHEDULE' THEN YEAR(ev.scheduleddate) - ELSE YEAR(ev.occurreddate) - END - """) + CASE + WHEN ev.status = 'SCHEDULE' THEN YEAR(ev.scheduleddate) + ELSE YEAR(ev.occurreddate) + END + """) .skipIndex(Skip.SKIP) .build(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 0837bc24ca0b..0c3e85fe9e0a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -28,6 +28,7 @@ package org.hisp.dhis.analytics.table; import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.hisp.dhis.analytics.table.model.Skip.SKIP; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getClosingParentheses; import static org.hisp.dhis.analytics.util.AnalyticsUtils.getColumnType; @@ -479,6 +480,82 @@ protected AnalyticsTableColumn getPartitionColumn() { .build(); } + /** + * Returns a column for the given data element. If the value type of the data element is {@link + * ValueType#ORGANISATION_UNIT}, an extra column will be included. + * + * @param dataElement the {@link DataElement}. + * @param withLegendSet indicates + * @return + */ + private List getColumnFromDataElement( + DataElement dataElement, boolean withLegendSet) { + List columns = new ArrayList<>(); + + DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); + String columnExpression = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + String selectExpression = getSelectExpression(dataElement.getValueType(), columnExpression); + String dataFilterClause = getDataFilterClause(dataElement.getUid(), dataElement.getValueType()); + String sql = getSelectForInsert(dataElement, selectExpression, dataFilterClause); + Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); + + if (dataElement.getValueType().isOrganisationUnit()) { + columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataFilterClause)); + } + + columns.add( + AnalyticsTableColumn.builder() + .name(dataElement.getUid()) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(dataType) + .selectExpression(sql) + .skipIndex(skipIndex) + .build()); + + return withLegendSet + ? getColumnFromDataElementWithLegendSet(dataElement, selectExpression, dataFilterClause) + : columns; + } + + private List getColumnFromOrgUnitDataElement( + DataElement dataElement, String dataFilterClause) { + List columns = new ArrayList<>(); + + String columnExpression = + sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); + String fromClause = + qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnExpression); + + if (isSpatialSupport()) { + String fromType = "ou.geometry " + fromClause; + String geoSql = getSelectForInsert(dataElement, fromType, dataFilterClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(GEOMETRY) + .selectExpression(geoSql) + .indexType(IndexType.GIST) + .build()); + } + + String fromTypeSql = "ou.name " + fromClause; + String ouNameSql = getSelectForInsert(dataElement, fromTypeSql, dataFilterClause); + + columns.add( + AnalyticsTableColumn.builder() + .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) + .columnType(AnalyticsColumnType.DYNAMIC) + .dataType(TEXT) + .selectExpression(ouNameSql) + .skipIndex(SKIP) + .build()); + + return columns; + } + private List getColumnFromTrackedEntityAttribute( TrackedEntityAttribute attribute, String numericClause, @@ -487,15 +564,16 @@ private List getColumnFromTrackedEntityAttribute( List columns = new ArrayList<>(); DataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); - String dataClause = + String selectExpression = getSelectExpressionForAttribute(attribute.getValueType(), "value"); + String dataExpression = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; - String select = getSelectClauseForTea(attribute.getValueType(), "value"); - String sql = selectForInsert(attribute, select, dataClause); + String sql = selectForInsert(attribute, selectExpression, dataExpression); Skip skipIndex = skipIndex(attribute.getValueType(), attribute.hasOptionSet()); if (attribute.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataClause)); + columns.addAll(getColumnsFromOrgUnitTrackedEntityAttribute(attribute, dataExpression)); } + columns.add( AnalyticsTableColumn.builder() .name(attribute.getUid()) @@ -512,7 +590,7 @@ private List getColumnFromTrackedEntityAttribute( private List getColumnFromTrackedEntityAttributeWithLegendSet( TrackedEntityAttribute attribute, String numericClause) { - String selectClause = getSelectClause(attribute.getValueType(), "value"); + String selectClause = getSelectExpression(attribute.getValueType(), "value"); String query = """ \s(select l.uid from ${maplegend} l \ @@ -545,45 +623,16 @@ private List getColumnFromTrackedEntityAttributeWithLegend .collect(toList()); } - private List getColumnFromDataElement( - DataElement dataElement, boolean withLegendSet) { - List columns = new ArrayList<>(); - - DataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); - String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); - String columnName = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - String select = getSelectClause(dataElement.getValueType(), columnName); - String sql = selectForInsert(dataElement, select, dataClause); - Skip skipIndex = skipIndex(dataElement.getValueType(), dataElement.hasOptionSet()); - - if (dataElement.getValueType().isOrganisationUnit()) { - columns.addAll(getColumnFromOrgUnitDataElement(dataElement, dataClause)); - } - columns.add( - AnalyticsTableColumn.builder() - .name(dataElement.getUid()) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(dataType) - .selectExpression(sql) - .skipIndex(skipIndex) - .build()); - - return withLegendSet - ? getColumnFromDataElementWithLegendSet(dataElement, select, dataClause) - : columns; - } - private List getColumnsFromOrgUnitTrackedEntityAttribute( - TrackedEntityAttribute attribute, String dataClause) { - final List columns = new ArrayList<>(); + TrackedEntityAttribute attribute, String dataFilterClause) { + List columns = new ArrayList<>(); - final String fromClause = + String fromClause = qualifyVariables("from ${organisationunit} ou where ou.uid = (select value"); if (isSpatialSupport()) { String fromType = "ou.geometry " + fromClause; - String geoSql = selectForInsert(attribute, fromType, dataClause); + String geoSql = selectForInsert(attribute, fromType, dataFilterClause); columns.add( AnalyticsTableColumn.builder() .name((attribute.getUid() + OU_GEOMETRY_COL_SUFFIX)) @@ -595,7 +644,7 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( } String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = selectForInsert(attribute, fromTypeSql, dataClause); + String ouNameSql = selectForInsert(attribute, fromTypeSql, dataFilterClause); columns.add( AnalyticsTableColumn.builder() @@ -609,75 +658,45 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( return columns; } - private List getColumnFromOrgUnitDataElement( - DataElement dataElement, String dataClause) { - final List columns = new ArrayList<>(); - - final String columnName = - sqlBuilder.jsonExtractNested("eventdatavalues", dataElement.getUid(), "value"); - - final String fromClause = - qualifyVariables("from ${organisationunit} ou where ou.uid = " + columnName); - - if (isSpatialSupport()) { - String fromType = "ou.geometry " + fromClause; - String geoSql = selectForInsert(dataElement, fromType, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((dataElement.getUid() + OU_GEOMETRY_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(GEOMETRY) - .selectExpression(geoSql) - .indexType(IndexType.GIST) - .build()); - } - - String fromTypeSql = "ou.name " + fromClause; - String ouNameSql = selectForInsert(dataElement, fromTypeSql, dataClause); - - columns.add( - AnalyticsTableColumn.builder() - .name((dataElement.getUid() + OU_NAME_COL_SUFFIX)) - .columnType(AnalyticsColumnType.DYNAMIC) - .dataType(TEXT) - .selectExpression(ouNameSql) - .skipIndex(SKIP) - .build()); - - return columns; - } - /** - * Creates a select statement for data element insertion. + * Creates a select statement for the given select expression. * - * @param dataElement The data element to create the select statement for - * @param fromType The SQL snippet for the "from" part of the query - * @param dataClause The data type related clause - * @return A SQL select expression for the data element + * @param dataElement the data element to create the select statement for. + * @param selectExpression the select expression. + * @param dataFilterClause the data filter clause. + * @return A SQL select expression for the data element. */ - private String selectForInsert(DataElement dataElement, String fromType, String dataClause) { - Map replacements = + private String getSelectForInsert( + DataElement dataElement, String selectExpression, String dataFilterClause) { + String sqlTemplate = + dataElement.getValueType().isOrganisationUnit() + ? "(select ${fromType} ${dataClause})${closingParentheses} as ${uid}" + : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${uid}"; + + Map variables = Map.of( "fromType", - fromType, + selectExpression, "dataClause", - dataClause, + dataFilterClause, "closingParentheses", - getClosingParentheses(fromType), - "dataElementUid", + getClosingParentheses(selectExpression), + "uid", quote(dataElement.getUid())); - String sqlTemplate = - dataElement.getValueType().isOrganisationUnit() - ? "(select ${fromType} ${dataClause})${closingParentheses} as ${dataElementUid}" - : "(select ${fromType} from ${event} where eventid=ev.eventid ${dataClause})${closingParentheses} as ${dataElementUid}"; - - return replaceQualify(sqlTemplate, replacements); + return replaceQualify(sqlTemplate, variables); } + /** + * Returns a list of columns. + * + * @param dataElement the {@link DataElement}. + * @param selectExpression the select expression. + * @param dataFilterClause the data filter clause. + * @return a list of {@link AnayticsTableColumn}. + */ private List getColumnFromDataElementWithLegendSet( - DataElement dataElement, String select, String dataClause) { + DataElement dataElement, String selectExpression, String dataFilterClause) { String query = """ (select l.uid from ${maplegend} l @@ -694,9 +713,9 @@ private List getColumnFromDataElementWithLegendSet( replaceQualify( query, Map.of( - "select", select, + "select", selectExpression, "legendSetId", String.valueOf(ls.getId()), - "dataClause", dataClause, + "dataClause", dataFilterClause, "column", column)); return AnalyticsTableColumn.builder() @@ -705,26 +724,43 @@ private List getColumnFromDataElementWithLegendSet( .selectExpression(sql) .build(); }) - .collect(toList()); + .toList(); } - private String getDataClause(String uid, ValueType valueType) { + /** + * For numeric and date value types, returns a data filter clause for checking whether the value + * is valid according to the value type. For other value types, returns the empty string. + * + * @param uid the identifier. + * @param valueType the {@link ValueType}. + * @return an expression for extracting a data value. + */ + private String getDataFilterClause(String uid, ValueType valueType) { if (valueType.isNumeric() || valueType.isDate()) { String regex = valueType.isNumeric() ? NUMERIC_LENIENT_REGEXP : DATE_REGEXP; - String jsonValue = sqlBuilder.jsonExtractNested("eventdatavalues", uid, "value"); + String jsonExpression = sqlBuilder.jsonExtractNested("eventdatavalues", uid, "value"); - return " and " + sqlBuilder.regexpMatch(jsonValue, "'" + regex + "'"); + return " and " + sqlBuilder.regexpMatch(jsonExpression, "'" + regex + "'"); } - return ""; + return EMPTY; } + /** + * Returns a list of years for which data exist. + * + * @param params the {@link AnalyticsTableUpdateParams}. + * @param program the {@link Program}. + * @param firstDataYear the first year to include. + * @param lastDataYear the last data year to include. + * @return a list of years for which data exist. + */ private List getDataYears( AnalyticsTableUpdateParams params, Program program, Integer firstDataYear, - Integer latestDataYear) { + Integer lastDataYear) { String fromDateClause = params.getFromDate() != null ? replace( @@ -734,7 +770,7 @@ private List getDataYears( eventDateExpression, "fromDate", toMediumDate(params.getFromDate()))) - : ""; + : EMPTY; String sql = replaceQualify( """ @@ -755,7 +791,7 @@ private List getDataYears( "programId", String.valueOf(program.getId()), "fromDateClause", fromDateClause, "firstDataYear", String.valueOf(firstDataYear), - "latestDataYear", String.valueOf(latestDataYear))); + "latestDataYear", String.valueOf(lastDataYear))); return jdbcTemplate.queryForList(sql, Integer.class); } @@ -764,8 +800,8 @@ private List getDataYears( * Retrieve years for partition tables. Year will become a partition key. The default return value * is the list with the recent year. * - * @param dataYears list of years coming from inner join of event and enrollment tables - * @return list of partition key values + * @param dataYears the list of years coming from inner join of event and enrollment tables. + * @return list of partition key values. */ private List getYearsForPartitionTable(List dataYears) { return ListUtils.mutableCopy(!dataYears.isEmpty() ? dataYears : List.of(Year.now().getValue())); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java index e23833d08d29..a4e1fc4b8997 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTrackedEntityAnalyticsTableManager.java @@ -255,7 +255,7 @@ private Stream getAllTrackedEntityAttributes( /** * Returns the select clause, potentially with a cast statement, based on the given value type. * (this method is an adapted version of {@link - * JdbcEventAnalyticsTableManager#getSelectClause(ValueType, String)}) + * JdbcEventAnalyticsTableManager#getSelectExpression(ValueType, String)}) * * @param valueType the value type to represent as database column type. */ diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index c4ee634d9821..8498baaaeac3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -219,7 +219,7 @@ public String coalesce(String expression, String defaultValue) { @Override public String jsonExtract(String column, String property) { - return "JSONExtractString(" + column + ", '" + property + "')"; + return String.format("JSONExtractString(%s, '%s')", column, property); } @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index b428875659cc..f7c9be3bd772 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -222,7 +222,7 @@ public String coalesce(String expression, String defaultValue) { @Override public String jsonExtract(String column, String property) { - return "json_unquote(json_extract(" + column + ", '$." + property + "'))"; + return String.format("json_unquote(json_extract(%s, '$.%s'))", column, property); } @Override