From d2285105aa6573163f0ddf2ca792f9f6bba66efa Mon Sep 17 00:00:00 2001 From: Github Actions Date: Fri, 13 Dec 2024 21:14:27 +0000 Subject: [PATCH 001/161] Version bump --- changelog.md | 6 +++++- gradle.properties | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 22a03ca98..e1a94949c 100644 --- a/changelog.md +++ b/changelog.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.0-beta25] - 2024-12-13 + ## [1.0.0-beta24] - 2024-12-02 ## [1.0.0-beta23] - 2024-11-23 @@ -61,7 +63,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.0.0-beta1] - 2024-06-14 -[Unreleased]: https://github.com/ortus-boxlang/BoxLang/compare/v1.0.0-beta24...HEAD +[Unreleased]: https://github.com/ortus-boxlang/BoxLang/compare/v1.0.0-beta25...HEAD + +[1.0.0-beta25]: https://github.com/ortus-boxlang/BoxLang/compare/v1.0.0-beta24...v1.0.0-beta25 [1.0.0-beta24]: https://github.com/ortus-boxlang/BoxLang/compare/v1.0.0-beta23...v1.0.0-beta24 diff --git a/gradle.properties b/gradle.properties index 7a39da2ba..82c01e8f8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -#Mon Dec 02 11:29:56 UTC 2024 +#Fri Dec 13 21:14:22 UTC 2024 antlrVersion=4.13.1 jdkVersion=21 -version=1.0.0-beta25 +version=1.0.0-beta26 From cc94238b8c974e67bdf0280e52f729c355a34fe8 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 13 Dec 2024 18:17:13 -0600 Subject: [PATCH 002/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 8 +- .../compiler/ast/sql/select/SQLJoinType.java | 11 +- .../ast/sql/select/expression/SQLColumn.java | 9 +- .../boxlang/compiler/parser/SQLParser.java | 2 + .../compiler/toolchain/SQLVisitor.java | 77 ++++++-- .../runtime/jdbc/qoq/QoQExecution.java | 9 - .../runtime/jdbc/qoq/QoQExecutionService.java | 72 ++----- .../jdbc/qoq/QoQIntersectionGenerator.java | 186 ++++++++++++++++++ .../ortus/boxlang/compiler/QoQParseTest.java | 34 ++-- 9 files changed, 306 insertions(+), 102 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 32fa9c279..7e11cacde 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -347,7 +347,11 @@ union: ; join_clause: - table (join_operator table join_constraint?)* + table join+ +; + +join: + join_operator table join_constraint? ; select_core: @@ -406,7 +410,7 @@ result_column: join_operator: // COMMA // | NATURAL_? ((LEFT_ | RIGHT_ | FULL_) OUTER_? | INNER_ | CROSS_)? JOIN_ - (INNER_)? JOIN_ + ((LEFT_ | RIGHT_ | FULL_) OUTER_? | INNER_ | CROSS_)? JOIN_ ; join_constraint: diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLJoinType.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLJoinType.java index a5b262ef4..f98ede6df 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLJoinType.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLJoinType.java @@ -19,18 +19,21 @@ public enum SQLJoinType { INNER, LEFT, RIGHT, - FULL; + FULL, + CROSS; public String getSymbol() { switch ( this ) { case INNER : return "INNER JOIN"; case LEFT : - return "LEFT JOIN"; + return "LEFT OUTER JOIN"; case RIGHT : - return "RIGHT JOIN"; + return "RIGHT OUTER JOIN"; case FULL : - return "FULL JOIN"; + return "FULL OUTER JOIN"; + case CROSS : + return "CROSS JOIN"; default : throw new IllegalStateException( "Unknown join type: " + this ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java index 82b297f98..76a625d50 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java @@ -113,10 +113,15 @@ public QueryColumnType getType( QoQExecution QoQExec ) { * Evaluate the expression */ public Object evaluate( QoQExecution QoQExec, int[] intersection ) { - var tableFinal = getTableFinal( QoQExec ); + var tableFinal = getTableFinal( QoQExec ); // System.out.println( "getting SQL column: " + name.getName() + " from table: " + tableFinal.getName() + " with index: " + tableFinal.getIndex() ); // System.out.println( "intersection: " + Arrays.toString( intersection ) ); - return QoQExec.getTableLookup().get( tableFinal ).getCell( name, intersection[ tableFinal.getIndex() ] - 1 ); + int rowNum = intersection[ tableFinal.getIndex() ]; + // This means an outer join matched nothing + if ( rowNum == 0 ) { + return null; + } + return QoQExec.getTableLookup().get( tableFinal ).getCell( name, rowNum - 1 ); } /** diff --git a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java index 62aadb961..c31f524ae 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java +++ b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java @@ -23,6 +23,7 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BOMInputStream; @@ -145,6 +146,7 @@ protected BoxNode parserFirstStage( InputStream stream, boolean classOrInterface addErrorListeners( lexer, parser ); parser.setErrorHandler( new BoxParserErrorStrategy() ); + parser.getInterpreter().setPredictionMode( PredictionMode.SLL ); ParserRuleContext parseTree = parser.parse(); // This must run FIRST before resetting the lexer diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index f4536ad62..c79a2599f 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -32,6 +32,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperation; import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperator; import ortus.boxlang.compiler.parser.SQLParser; +import ortus.boxlang.parser.antlr.SQLGrammar; import ortus.boxlang.parser.antlr.SQLGrammar.ExprContext; import ortus.boxlang.parser.antlr.SQLGrammar.Literal_valueContext; import ortus.boxlang.parser.antlr.SQLGrammar.Ordering_termContext; @@ -167,27 +168,32 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { List groupBys = null; SQLExpression having = null; + TableContext firstTable; if ( !ctx.table().isEmpty() ) { - var firstTable = ctx.table().get( 0 ); - table = ( SQLTable ) visit( firstTable ); - } - - // TODO: handle joins - // Treat "FROM table1, table2" as a join - if ( ctx.table() != null && ctx.table().size() > 1 ) { - joins = new ArrayList(); - for ( int i = 1; i < ctx.table().size(); i++ ) { - var tableCtx = ctx.table().get( i ); - SQLTable joinTable = ( SQLTable ) visit( tableCtx ); - joins.add( new SQLJoin( SQLJoinType.INNER, joinTable, null, tools.getPosition( tableCtx ), tools.getSourceText( tableCtx ) ) ); + firstTable = ctx.table().get( 0 ); + table = ( SQLTable ) visit( firstTable ); + + if ( ctx.table().size() > 1 ) { + // from table1, table2 is treated as a join with no `on` clause + joins = new ArrayList(); + for ( int i = 1; i < ctx.table().size(); i++ ) { + var tableCtx = ctx.table().get( i ); + SQLTable joinTable = ( SQLTable ) visit( tableCtx ); + joins.add( new SQLJoin( SQLJoinType.FULL, joinTable, null, tools.getPosition( tableCtx ), tools.getSourceText( tableCtx ) ) ); + } } + } else if ( ctx.join_clause() != null ) { + firstTable = ctx.join_clause().table(); + table = ( SQLTable ) visit( firstTable ); + joins = buildJoins( ctx.join_clause(), table ); } // limit before order by, can have one per unioned table if ( ctx.limit_stmt() != null ) { limit = NUMERIC_LITERAL( ctx.limit_stmt().NUMERIC_LITERAL() ); } - // each + + // each select can have a top if ( ctx.top() != null ) { limit = NUMERIC_LITERAL( ctx.top().NUMERIC_LITERAL() ); } @@ -200,8 +206,6 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { // TODO: having - // TODO: handle additional tables as joins - // Do this after all joins above so we know the tables available to us final SQLTable finalTable = table; final List finalJoins = joins; @@ -221,6 +225,49 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { return result; } + public List buildJoins( SQLGrammar.Join_clauseContext ctx, SQLTable table ) { + var joins = new ArrayList(); + for ( var joinCtx : ctx.join() ) { + var pos = tools.getPosition( joinCtx ); + var src = tools.getSourceText( joinCtx ); + var joinTable = ( SQLTable ) visit( joinCtx.table() ); + var typeCtx = joinCtx.join_operator(); + boolean hasOn = joinCtx.join_constraint() != null; + String joinText = tools.getSourceText( typeCtx ); + // If left, right, full, or cross are not specified, we default to inner + SQLJoinType type = SQLJoinType.INNER; + if ( typeCtx.LEFT_() != null ) { + if ( !hasOn ) { + tools.reportError( "[" + joinText + "] must have an ON clause", tools.getPosition( typeCtx ) ); + } + type = SQLJoinType.LEFT; + } else if ( typeCtx.RIGHT_() != null ) { + if ( !hasOn ) { + tools.reportError( "[" + joinText + "] must have an ON clause", tools.getPosition( typeCtx ) ); + } + type = SQLJoinType.RIGHT; + } else if ( typeCtx.FULL_() != null ) { + if ( !hasOn ) { + tools.reportError( "[" + joinText + "] must have an ON clause", tools.getPosition( typeCtx ) ); + } + type = SQLJoinType.FULL; + } else if ( typeCtx.CROSS_() != null ) { + if ( hasOn ) { + tools.reportError( "[" + joinText + "] cannot have an ON clause", tools.getPosition( typeCtx ) ); + } + type = SQLJoinType.CROSS; + } + // Leave expression null here. I need to get the JOIN into the list of joins FIRST so the expression + // visitors can correctly match the table names. Well add the expression later + SQLJoin thisJoin = new SQLJoin( type, joinTable, null, pos, src ); + joins.add( thisJoin ); + if ( hasOn ) { + thisJoin.setOn( visitExpr( joinCtx.join_constraint().expr(), table, joins ) ); + } + } + return joins; + } + /** * Visit the class or interface context to generate the AST node for the * top level node diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java index 335ea32bf..c7672e0f9 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java @@ -35,7 +35,6 @@ public class QoQExecution { public Map resultColumns = null; public Map tableLookup; public List params; - public int totalCombinations = 0; public List orderByColumns = null; public Set additionalColumns = null; @@ -58,14 +57,6 @@ public static QoQExecution of( SQLSelectStatement select, Map t return new QoQExecution( select, tableLookup, params ); } - public void setTotalCombinations( int totalCombinations ) { - this.totalCombinations = totalCombinations; - } - - public int getTotalCombinations() { - return totalCombinations; - } - public SQLSelectStatement getSelect() { return select; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 0dcec90a4..3ce06ec8e 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -16,14 +16,11 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Stream; import ortus.boxlang.compiler.ast.sql.SQLNode; import ortus.boxlang.compiler.ast.sql.select.SQLJoin; @@ -113,13 +110,19 @@ public static Query executeSelect( IBoxContext context, SQLSelectStatement selec statement instanceof QoQPreparedStatement qp ? qp.getParameters() : null ); - var intersections = createCartesianStream( QoQExec ); + var intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); Map resultColumns = calculateResultColumns( QoQExec ); calculateOrderBys( QoQExec ); Query target = buildTargetQuery( QoQExec ); + // print out arrays + /* + * intersections.forEach( i -> System.out.println( Arrays.toString( i ) ) ); + * if ( true ) + * return target; + */ // Process one select // TODO: refactor this out SQLSelect select = selectStatement.getSelect(); @@ -148,12 +151,13 @@ public static Query executeSelect( IBoxContext context, SQLSelectStatement selec // enforce top/limit for this select. This would be a "top N" clause in the select or a "limit N" clause BEFORE the order by, which // could exist or all selects in a union. if ( canEarlyLimit && thisSelectLimit > -1 ) { - intersections = intersections.limit( Math.min( thisSelectLimit, QoQExec.getTotalCombinations() ) ); + intersections = intersections.limit( thisSelectLimit ); } // 1-based index! intersections.forEach( intersection -> { + // System.out.println( Arrays.toString( intersection ) ); // Evaluate the where expression if ( where == null || ( Boolean ) where.evaluate( QoQExec, intersection ) ) { Object[] values = new Object[ resultColumns.size() ]; @@ -262,16 +266,19 @@ private static Map calculateResultColumns( QoQExecution for ( SQLResultColumn resultColumn : QoQExec.select.getSelect().getResultColumns() ) { // For *, expand all columns in the query if ( resultColumn.isStarExpression() ) { + // The same table joined more than once will still have separate SQLTable instances in the AST. + // If we have a specific alias such as t.* this will still match since the correct SQLTable reference will be associated with the result column SQLTable starTable = ( ( SQLStarExpression ) resultColumn.getExpression() ).getTable(); Key tableName = starTable == null ? null : starTable.getName(); - var matchingTables = QoQExec.tableLookup.keySet().stream().filter( t -> tableName == null || tableName.equals( t.getName() ) ) + + var matchingTables = QoQExec.tableLookup.keySet().stream().filter( t -> starTable == null || starTable == t ) .toList(); + if ( matchingTables.isEmpty() ) { throw new DatabaseException( "The table alias [" + tableName + "] in the result column [" + resultColumn.getSourceText() + "] is does not match a table." ); } matchingTables.stream().forEach( t -> { - System.out.println( "Expanding * for table: " + t.getName() ); var thisTable = QoQExec.tableLookup.get( t ); for ( Key key : thisTable.getColumns().keySet() ) { resultColumns.put( key, @@ -307,57 +314,6 @@ private static Query getSourceQuery( IBoxContext context, String tableVarName ) } } - /** - * Create all possible intercections of the tables as a stream of int arrays - */ - public static Stream createCartesianStream( QoQExecution QoQExec ) { - Map tableLookup = QoQExec.tableLookup; - int numTables = tableLookup.size(); - // I hold the number of rows in each corresponding query. table lookup is a linked array, so it contains the tables in encounter order from the query - int[] rowCounts = new int[ numTables ]; - // Tracks what row we're on for each table - int[] current = new int[ numTables ]; - Arrays.fill( current, 1 ); - int i = 0; - for ( Query table : tableLookup.values() ) { - rowCounts[ i++ ] = table.size(); - } - - // Total number of combinations, which is all recordcounts multiplied together - int totalCombinations = Arrays.stream( rowCounts ).reduce( 1, ( a, b ) -> a * b ); - QoQExec.setTotalCombinations( totalCombinations ); - Supplier supplier = () -> { - int[] indices = Arrays.copyOf( current, numTables ); - // FWIW, the last time this supplier runs, we will increment nothing - for ( int j = 0; j < numTables; j++ ) { - // Stop incrementing each table when we reach the end of the table - if ( current[ j ] < rowCounts[ j ] ) { - current[ j ]++; - // As soon as we increment any table, we're done. - // This ensure we hit every possible combo by only changing one index at a time. - break; - } else { - // The first column always resets, all other colmns reset if their previous column just reset - if ( j == 0 || current[ j - 1 ] == 1 ) { - current[ j ] = 1; - } - } - } - return indices; - }; - - // Limit the stream to the total number of combos - Stream theStream = Stream.generate( supplier ).limit( totalCombinations ); - - // Tweak this based on size of intersections to process - if ( totalCombinations > 50 ) { - // The supplier above MUST be called in order, so we can't parallelize it due to stupid Java behaviors of pre-loading supplier calls. - // Collect the values and then create a new stream from the list - theStream = theStream.toList().stream().parallel(); - } - return theStream; - } - public record TypedResultColumn( QueryColumnType type, SQLResultColumn resultColumn ) { public static TypedResultColumn of( QueryColumnType type, SQLResultColumn resultColumn ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java new file mode 100644 index 000000000..515fb7883 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java @@ -0,0 +1,186 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import ortus.boxlang.compiler.ast.sql.select.SQLJoin; +import ortus.boxlang.compiler.ast.sql.select.SQLJoinType; +import ortus.boxlang.compiler.ast.sql.select.SQLTable; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.types.Query; + +/** + * I handle the generation of the intersection of tables in a QoQ statement + */ +public class QoQIntersectionGenerator { + + /** + * Create all possible intersections of the tables as a stream of int arrays + */ + public static Stream createIntersectionStream( QoQExecution QoQExec ) { + Map tableLookup = QoQExec.tableLookup; + Query firstTable = tableLookup.get( QoQExec.select.getSelect().getTable() ); + // This is just an estimation of the total number of combinations + // Streams make it hard to get an exact count without collecting the stream, so we'll just guess + int totalCombinations = firstTable.size(); + // stream of int arrays containing the index of the row in each table + // use rangeClosed to include 1 through query size + Stream theStream = IntStream.rangeClosed( 1, firstTable.size() ) + .mapToObj( i -> new int[] { i } ); + + // If we have one or more joins, we need to create a stream of all possible intersections + if ( QoQExec.select.getSelect().getJoins() != null ) { + for ( SQLJoin thisJoin : QoQExec.select.getSelect().getJoins() ) { + var joinType = thisJoin.getType(); + var joinTable = tableLookup.get( thisJoin.getTable() ); + var joinOn = thisJoin.getOn(); + + if ( joinType.equals( SQLJoinType.CROSS ) || joinType.equals( SQLJoinType.INNER ) ) { + theStream = handleCrossOrInnerJoin( theStream, joinTable, joinOn, QoQExec ); + totalCombinations *= joinTable.size(); + } else if ( joinType.equals( SQLJoinType.LEFT ) ) { + theStream = handleLeftJoin( theStream, joinTable, joinOn, QoQExec ); + totalCombinations *= joinTable.size(); + } else if ( joinType.equals( SQLJoinType.RIGHT ) ) { + theStream = handleRightJoin( theStream, joinTable, joinOn, QoQExec ); + totalCombinations *= joinTable.size(); + } else if ( joinType.equals( SQLJoinType.FULL ) ) { + theStream = handleFullOuterJoin( theStream, joinTable, joinOn, QoQExec ); + totalCombinations *= joinTable.size(); + } + } + } + + // Tweak this based on size of intersections to process + if ( totalCombinations > 50 ) { + theStream = theStream.parallel(); + } + return theStream; + } + + private static Stream handleCrossOrInnerJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + theStream = theStream.flatMap( i -> IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { + int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); + newIntersection[ i.length ] = j; + return newIntersection; + } ) ); + if ( joinOn != null ) { + theStream = theStream.filter( i -> ( Boolean ) joinOn.evaluate( QoQExec, i ) ); + } + return theStream; + } + + private static Stream handleLeftJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + return theStream.flatMap( i -> { + Stream newStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { + int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); + newIntersection[ i.length ] = j; + return newIntersection; + } ).filter( j -> ( Boolean ) joinOn.evaluate( QoQExec, j ) ); + List newStreamList = newStream.collect( Collectors.toList() ); + if ( newStreamList.isEmpty() ) { + int[] leftOnlyIntersection = Arrays.copyOf( i, i.length + 1 ); + leftOnlyIntersection[ i.length ] = 0; // 0 indicates no match in the right table + return Stream.of( leftOnlyIntersection ); + } + return newStreamList.stream(); + } ); + } + + private static Stream handleRightJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream + Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); + return rightStream.flatMap( j -> { + Stream newStream = leftRows.stream().map( i -> { + int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); + newIntersection[ i.length ] = j[ 0 ]; + return newIntersection; + } ).filter( joint -> ( Boolean ) joinOn.evaluate( QoQExec, joint ) ); + List newStreamList = newStream.collect( Collectors.toList() ); + if ( newStreamList.isEmpty() ) { + int[] rightOnlyIntersection = new int[ leftRows.get( 0 ).length + 1 ]; + rightOnlyIntersection[ rightOnlyIntersection.length - 1 ] = j[ 0 ]; + for ( int k = 0; k < rightOnlyIntersection.length - 1; k++ ) { + rightOnlyIntersection[ k ] = 0; // 0 indicates no match in the left table + } + return Stream.of( rightOnlyIntersection ); + } + return newStreamList.stream(); + } ); + } + + private static Stream handleFullOuterJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream + Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); + + // Process LEFT JOIN logic + Stream leftJoinStream = leftRows.stream().flatMap( i -> { + Stream newStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { + int[] newIntersection = Arrays + .copyOf( + i, + i.length + + 1 ); + newIntersection[ i.length ] = j; + return newIntersection; + } ) + .filter( j -> ( Boolean ) joinOn.evaluate( QoQExec, j ) ); + List newStreamList = newStream.collect( Collectors.toList() ); + if ( newStreamList.isEmpty() ) { + int[] leftOnlyIntersection = Arrays.copyOf( i, i.length + 1 ); + leftOnlyIntersection[ i.length ] = 0; // 0 indicates no match in the right table + return Stream.of( leftOnlyIntersection ); + } + return newStreamList.stream(); + } ); + + // Process RIGHT JOIN logic + Stream rightJoinStream = rightStream.flatMap( j -> { + Stream newStream = leftRows.stream().map( i -> { + int[] newIntersection = Arrays + .copyOf( + i, + i.length + + 1 ); + newIntersection[ i.length ] = j[ 0 ]; + return newIntersection; + } ) + .filter( joint -> ( Boolean ) joinOn.evaluate( QoQExec, joint ) ); + List newStreamList = newStream.collect( Collectors.toList() ); + if ( newStreamList.isEmpty() ) { + int[] rightOnlyIntersection = new int[ leftRows.get( 0 ).length + 1 ]; + rightOnlyIntersection[ rightOnlyIntersection.length - 1 ] = j[ 0 ]; + for ( int k = 0; k < rightOnlyIntersection.length - 1; k++ ) { + rightOnlyIntersection[ k ] = 0; // 0 indicates no match in the left table + } + return Stream.of( rightOnlyIntersection ); + } + return newStreamList.stream(); + } ); + + // Combine LEFT JOIN and RIGHT JOIN results and remove duplicates using a Set + Set> seen = new HashSet<>(); + return Stream.concat( leftJoinStream, rightJoinStream ) + .filter( arr -> seen.add( Arrays.stream( arr ).boxed().collect( Collectors.toList() ) ) ); // Remove duplicates + } +} diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 4f8c428ba..4ee6d134b 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -82,18 +82,28 @@ public void testMetadataVisitor() { public void testRunQoQ() { instance.executeSource( """ - qryEmployees = queryNew( "name,age,dept", "varchar,integer,varchar", [["brad",44,"IT"],["luis",43,"Exec"],["Jon",45,"IT"]] ) - qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200]] ) - q = queryExecute( " - select e.*, d.code - from qryEmployees e, qryDept d - where e.dept = d.name - ", - [], - { dbType : "query" } - ); - println( q ) - """, + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200],["Janitor",200]] ) + q = queryExecute( " + select e.*, s.name as supName, d.name as deptname + from qryEmployees e + inner join qryEmployees s on e.supervisor = s.name + full join qryDept d on e.dept = d.name + where d.name in ('IT','HR') + ", + [], + { dbType : "query" } + ); + println( q ) + """, context ); } From 17a25d37bd99cdf390a7ef2e9120737f4d3bc3d9 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 13 Dec 2024 18:44:53 -0600 Subject: [PATCH 003/161] BL-823 --- .../boxlang/runtime/jdbc/qoq/QoQExecutionService.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 3ce06ec8e..a38326b25 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import ortus.boxlang.compiler.ast.sql.SQLNode; import ortus.boxlang.compiler.ast.sql.select.SQLJoin; @@ -101,18 +102,22 @@ public static Query executeSelect( IBoxContext context, SQLSelectStatement selec String tableVarName = table.getVariableName(); tableLookup.put( table, getSourceQuery( context, tableVarName ) ); } + hasTable = true; } // This holds the AST and the runtime values for the query - QoQExecution QoQExec = QoQExecution.of( + QoQExecution QoQExec = QoQExecution.of( selectStatement, tableLookup, statement instanceof QoQPreparedStatement qp ? qp.getParameters() : null ); - var intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); + Stream intersections = null; + if ( hasTable ) { + intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); + } - Map resultColumns = calculateResultColumns( QoQExec ); + Map resultColumns = calculateResultColumns( QoQExec ); calculateOrderBys( QoQExec ); Query target = buildTargetQuery( QoQExec ); From 393b93af1b9501f971ba20ab46b8e889761d0260 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 14 Dec 2024 16:44:37 -0600 Subject: [PATCH 004/161] BL-848 --- .../phase3/ClassLocatorInStaticInitializer.bx | 5 +++++ src/test/java/TestCases/phase3/ClassTest.java | 12 ++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx diff --git a/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx new file mode 100644 index 000000000..c9a9d8b48 --- /dev/null +++ b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx @@ -0,0 +1,5 @@ +class { + static { + createObject( "java", "java.io.File" ) + } +} \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 32f645c27..5450c41f6 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1555,4 +1555,16 @@ public void testMixinsPublic() { assertThat( variables.get( "result" ) ).isEqualTo( "mixed up" ); } + + @DisplayName( "class locator usage in static initializer" ) + @Test + @Disabled( "BL-848" ) + public void testClassLocatorInStaticInitializer() { + instance.executeSource( + """ + new src.test.java.TestCases.phase3.ClassLocatorInStaticInitializer() + """, + context ); + } + } From 58c3a794d69df32753dde7ea137e6bfe31ce28a6 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 14 Dec 2024 18:23:42 -0600 Subject: [PATCH 005/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 27 +- .../ast/sql/select/SQLSelectStatement.java | 10 +- .../compiler/ast/sql/select/SQLTable.java | 91 +---- .../ast/sql/select/SQLTableSubQuery.java | 83 +++++ .../ast/sql/select/SQLTableVariable.java | 116 +++++++ .../compiler/ast/sql/select/SQLUnion.java | 98 ++++++ .../compiler/ast/sql/select/SQLUnionType.java | 33 ++ .../ast/sql/select/expression/SQLColumn.java | 12 +- .../sql/select/expression/SQLExpression.java | 10 +- .../sql/select/expression/SQLFunction.java | 10 +- .../ast/sql/select/expression/SQLParam.java | 14 +- .../sql/select/expression/SQLParenthesis.java | 10 +- .../select/expression/SQLStarExpression.java | 4 +- .../expression/literal/SQLBooleanLiteral.java | 6 +- .../expression/literal/SQLNullLiteral.java | 4 +- .../expression/literal/SQLNumberLiteral.java | 8 +- .../expression/literal/SQLStringLiteral.java | 6 +- .../operation/SQLBetweenOperation.java | 6 +- .../operation/SQLBinaryOperation.java | 18 +- .../expression/operation/SQLInOperation.java | 6 +- .../operation/SQLUnaryOperation.java | 16 +- .../compiler/toolchain/SQLVisitor.java | 108 ++++-- .../jdbc/qoq/QoQAggregateFunctionDef.java | 2 +- .../runtime/jdbc/qoq/QoQExecutionService.java | 320 ++++++++---------- .../runtime/jdbc/qoq/QoQFunctionService.java | 8 +- .../jdbc/qoq/QoQIntersectionGenerator.java | 16 +- .../jdbc/qoq/QoQPreparedStatement.java | 2 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 186 ++++++++++ ....java => QoQSelectStatementExecution.java} | 66 ++-- .../runtime/jdbc/qoq/QoQStatement.java | 2 +- .../jdbc/qoq/functions/aggregate/Max.java | 4 +- .../ortus/boxlang/compiler/QoQParseTest.java | 39 +++ 32 files changed, 928 insertions(+), 413 deletions(-) create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableSubQuery.java create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnion.java create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnionType.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java rename src/main/java/ortus/boxlang/runtime/jdbc/qoq/{QoQExecution.java => QoQSelectStatementExecution.java} (51%) diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 7e11cacde..7a0ee42bb 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -343,20 +343,20 @@ select_stmt: ; union: - UNION_ ALL_? select_core + UNION_ ALL_? DISTINCT_? select_core ; join_clause: - table join+ + table_or_subquery join+ ; join: - join_operator table join_constraint? + join_operator table_or_subquery join_constraint? ; select_core: SELECT_ top? (DISTINCT_ /*| ALL_*/)? result_column (COMMA result_column)* ( - FROM_ (table (COMMA table)* | join_clause) + FROM_ (table_or_subquery (COMMA table_or_subquery)* | join_clause) )? (WHERE_ whereExpr = expr)? ( GROUP_ BY_ groupByExpr += expr (COMMA groupByExpr += expr)* ( HAVING_ havingExpr = expr @@ -388,17 +388,13 @@ table: (schema_name DOT)? table_name (AS_? table_alias)? ; -table_or_subquery: ( - (schema_name DOT)? table_name (AS_? table_alias)? ( - INDEXED_ BY_ index_name - | NOT_ INDEXED_ - )? - ) - | (schema_name DOT)? table_function_name OPEN_PAR expr (COMMA expr)* CLOSE_PAR ( - AS_? table_alias - )? - | OPEN_PAR (table_or_subquery (COMMA table_or_subquery)* | join_clause) CLOSE_PAR - | OPEN_PAR select_stmt CLOSE_PAR (AS_? table_alias)? +subquery: + OPEN_PAR select_stmt CLOSE_PAR AS_? table_alias +; + +table_or_subquery: + table + | subquery ; result_column: @@ -580,6 +576,7 @@ expr_asc_desc: ; //TODO BOTH OF THESE HAVE TO BE REWORKED TO FOLLOW THE SPEC + initial_select: select_stmt ; diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLSelectStatement.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLSelectStatement.java index ea3520091..8205a99f8 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLSelectStatement.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLSelectStatement.java @@ -32,7 +32,7 @@ public class SQLSelectStatement extends SQLStatement { private SQLSelect select; - private List unions; + private List unions; private List orderBys; private SQLNumberLiteral limit; @@ -42,7 +42,7 @@ public class SQLSelectStatement extends SQLStatement { * @param position position of the statement in the source code * @param sourceText source code of the statement */ - public SQLSelectStatement( SQLSelect select, List unions, List orderBys, SQLNumberLiteral limit, Position position, + public SQLSelectStatement( SQLSelect select, List unions, List orderBys, SQLNumberLiteral limit, Position position, String sourceText ) { super( position, sourceText ); setSelect( select ); @@ -72,7 +72,7 @@ public SQLSelect getSelect() { /** * Set the UNIONed SELECT statements */ - public void setUnions( List unions ) { + public void setUnions( List unions ) { replaceChildren( this.unions, unions ); this.unions = unions; if ( unions != null ) { @@ -83,7 +83,7 @@ public void setUnions( List unions ) { /** * Get the UNIONed SELECT statements */ - public List getUnions() { + public List getUnions() { return unions; } @@ -151,7 +151,7 @@ public Map toMap() { map.put( "select", select.toMap() ); if ( unions != null ) { - map.put( "unions", unions.stream().map( SQLSelect::toMap ).toList() ); + map.put( "unions", unions.stream().map( SQLUnion::toMap ).toList() ); } else { map.put( "unions", null ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTable.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTable.java index 4bff3a9fd..096d4de5a 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTable.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTable.java @@ -14,30 +14,21 @@ */ package ortus.boxlang.compiler.ast.sql.select; -import java.util.Map; - -import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.SQLNode; -import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; -import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.scopes.Key; /** - * Abstract Node class representing SQL table declaration + * Class representing SQL table value. I may be a reference to a variable name or a sub-select */ -public class SQLTable extends SQLNode { - - private String schema; +public abstract class SQLTable extends SQLNode { - private Key name; - - private Key alias; + protected Key alias; /** * Encounter order of the table in the query. This should match the position of the table in the tableLookup map later */ - private int index; + protected int index; /** * Constructor @@ -45,42 +36,12 @@ public class SQLTable extends SQLNode { * @param position position of the statement in the source code * @param sourceText source code of the statement */ - public SQLTable( String schema, String name, String alias, int index, Position position, String sourceText ) { + public SQLTable( String alias, int index, Position position, String sourceText ) { super( position, sourceText ); - setSchema( schema ); - setName( name ); setAlias( alias ); setIndex( index ); } - /** - * Get the schema name - */ - public String getSchema() { - return schema; - } - - /** - * Set the schema name - */ - public void setSchema( String schema ) { - this.schema = schema; - } - - /** - * Get the table name - */ - public Key getName() { - return name; - } - - /** - * Set the table name - */ - public void setName( String name ) { - this.name = Key.of( name ); - } - /** * Get the table alias */ @@ -109,46 +70,6 @@ public void setIndex( int index ) { this.index = index; } - public boolean isCalled( Key name ) { - return this.name.equals( name ) || ( alias != null && alias.equals( name ) ); - } - - public String getVariableName() { - if ( schema != null ) { - return schema + "." + name.getName(); - } else { - return name.getName(); - } - } - - @Override - public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); - } - - @Override - public BoxNode accept( ReplacingBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); - } - - @Override - public Map toMap() { - Map map = super.toMap(); - - if ( schema != null ) { - map.put( "schema", schema ); - } else { - map.put( "schema", null ); - } - map.put( "name", name.getName() ); - if ( alias != null ) { - map.put( "alias", alias.getName() ); - } else { - map.put( "alias", null ); - } - return map; - } + abstract public boolean isCalled( Key name ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableSubQuery.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableSubQuery.java new file mode 100644 index 000000000..8cd6b8b41 --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableSubQuery.java @@ -0,0 +1,83 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select; + +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.scopes.Key; + +/** + * Class representing SQL table as a sub query + */ +public class SQLTableSubQuery extends SQLTable { + + private SQLSelectStatement selectStatement; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLTableSubQuery( SQLSelectStatement selectStatement, String alias, int index, Position position, String sourceText ) { + super( alias, index, position, sourceText ); + setSelectStatement( selectStatement ); + } + + /** + * Get the select statement + */ + public SQLSelectStatement getSelectStatement() { + return selectStatement; + } + + /** + * Set the select statement + */ + public void setSelectStatement( SQLSelectStatement selectStatement ) { + replaceChildren( this.selectStatement, selectStatement ); + this.selectStatement = selectStatement; + this.selectStatement.setParent( this ); + } + + public boolean isCalled( Key name ) { + return alias.equals( name ); + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + map.put( "selectStatement", selectStatement.toMap() ); + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java new file mode 100644 index 000000000..0cb4f2216 --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java @@ -0,0 +1,116 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select; + +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.scopes.Key; + +/** + * Class representing SQL table as a variable name + */ +public class SQLTableVariable extends SQLTable { + + private String schema; + + private Key name; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLTableVariable( String schema, String name, String alias, int index, Position position, String sourceText ) { + super( alias, index, position, sourceText ); + setSchema( schema ); + setName( name ); + } + + /** + * Get the schema name + */ + public String getSchema() { + return schema; + } + + /** + * Set the schema name + */ + public void setSchema( String schema ) { + this.schema = schema; + } + + /** + * Get the table name + */ + public Key getName() { + return name; + } + + /** + * Set the table name + */ + public void setName( String name ) { + this.name = Key.of( name ); + } + + public boolean isCalled( Key name ) { + return this.name.equals( name ) || ( alias != null && alias.equals( name ) ); + } + + public String getVariableName() { + if ( schema != null ) { + return schema + "." + name.getName(); + } else { + return name.getName(); + } + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + if ( schema != null ) { + map.put( "schema", schema ); + } else { + map.put( "schema", null ); + } + map.put( "name", name.getName() ); + if ( alias != null ) { + map.put( "alias", alias.getName() ); + } else { + map.put( "alias", null ); + } + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnion.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnion.java new file mode 100644 index 000000000..fa7c9672a --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnion.java @@ -0,0 +1,98 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select; + +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.sql.SQLNode; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; + +/** + * Abstract Node class representing SQL UNION statement + */ +public class SQLUnion extends SQLNode { + + private SQLSelect select; + + private SQLUnionType type; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLUnion( SQLSelect select, SQLUnionType type, Position position, String sourceText ) { + super( position, sourceText ); + setSelect( select ); + setType( type ); + } + + /** + * Get the SELECT statement + */ + public SQLSelect getSelect() { + return select; + } + + /** + * Set the SELECT statement + */ + public void setSelect( SQLSelect select ) { + replaceChildren( this.select, select ); + this.select = select; + select.setParent( this ); + } + + /** + * Get the type of the UNION + */ + public SQLUnionType getType() { + return type; + } + + /** + * Set the type of the UNION + */ + public void setType( SQLUnionType type ) { + this.type = type; + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + map.put( "select", select.toMap() ); + map.put( "type", enumToMap( type ) ); + + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnionType.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnionType.java new file mode 100644 index 000000000..36d9a8687 --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLUnionType.java @@ -0,0 +1,33 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select; + +public enum SQLUnionType { + + ALL, + DISTINCT; + + public String getSymbol() { + switch ( this ) { + case ALL : + return "UNION ALL"; + case DISTINCT : + return "UNION"; + default : + throw new IllegalStateException( "Unknown union type: " + this ); + } + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java index 76a625d50..9e07193a8 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java @@ -22,7 +22,7 @@ import ortus.boxlang.compiler.ast.sql.select.SQLTable; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -79,7 +79,7 @@ public SQLTable getTable() { /** * Get the table, performing runtime lookup if necessary */ - public SQLTable getTableFinal( QoQExecution QoQExec ) { + public SQLTable getTableFinal( QoQSelectExecution QoQExec ) { var t = getTable(); if ( t != null ) { return t; @@ -105,14 +105,14 @@ public void setTable( SQLTable table ) { /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return QoQExec.getTableLookup().get( getTableFinal( QoQExec ) ).getColumns().get( name ).getType(); } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { var tableFinal = getTableFinal( QoQExec ); // System.out.println( "getting SQL column: " + name.getName() + " from table: " + tableFinal.getName() + " with index: " + tableFinal.getIndex() ); // System.out.println( "intersection: " + Arrays.toString( intersection ) ); @@ -131,7 +131,7 @@ public Object evaluate( QoQExecution QoQExec, int[] intersection ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return getType( QoQExec ) == QueryColumnType.BIT; } @@ -142,7 +142,7 @@ public boolean isBoolean( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return numericTypes.contains( getType( QoQExec ) ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java index c0893767a..bf17f5a10 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java @@ -16,7 +16,7 @@ import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.SQLNode; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.QueryColumnType; /** @@ -48,7 +48,7 @@ public boolean isLiteral() { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return false; } @@ -59,14 +59,14 @@ public boolean isBoolean( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return false; } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { if ( isBoolean( QoQExec ) ) { return QueryColumnType.BIT; } @@ -76,6 +76,6 @@ public QueryColumnType getType( QoQExecution QoQExec ) { /** * Evaluate the expression */ - public abstract Object evaluate( QoQExecution QoQExec, int[] intersection ); + public abstract Object evaluate( QoQSelectExecution QoQExec, int[] intersection ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java index 969fe946b..d39197b3c 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java @@ -22,9 +22,9 @@ import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; import ortus.boxlang.runtime.jdbc.qoq.QoQFunctionService; import ortus.boxlang.runtime.jdbc.qoq.QoQFunctionService.QoQFunction; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -97,7 +97,7 @@ public void setArguments( List arguments ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return getType( QoQExec ) == QueryColumnType.BIT; } @@ -108,7 +108,7 @@ public boolean isBoolean( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return numericTypes.contains( getType( QoQExec ) ); } @@ -122,14 +122,14 @@ public boolean isAggregate() { /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return QoQFunctionService.getFunction( name ).returnType(); } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { QoQFunction function = QoQFunctionService.getFunction( name ); if ( function.requiredParams() > arguments.size() ) { throw new RuntimeException( diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java index a1a33b214..ed930ad5b 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.QueryColumnType; /** @@ -91,7 +91,7 @@ public void setIndex( int index ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return getType( QoQExec ) == QueryColumnType.BIT; } @@ -102,22 +102,22 @@ public boolean isBoolean( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return numericTypes.contains( getType( QoQExec ) ); } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { - return QueryColumnType.fromSQLType( QoQExec.getParams().get( index ).type() ); + public QueryColumnType getType( QoQSelectExecution QoQExec ) { + return QueryColumnType.fromSQLType( QoQExec.getSelectStatementExecution().getParams().get( index ).type() ); } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { - return QoQExec.getParams().get( index ).value(); + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { + return QoQExec.getSelectStatementExecution().getParams().get( index ).value(); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java index be9041f3e..287bcf40c 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java @@ -20,7 +20,7 @@ import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.QueryColumnType; /** @@ -64,7 +64,7 @@ public void setExpression( SQLExpression expression ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return expression.isBoolean( QoQExec ); } @@ -75,21 +75,21 @@ public boolean isBoolean( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return expression.isNumeric( QoQExec ); } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return expression.getType( QoQExec ); } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return expression.evaluate( QoQExec, intersection ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java index b6e4e0800..1719d9e4d 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.sql.select.SQLTable; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; /** @@ -60,7 +60,7 @@ public void setTable( SQLTable table ) { /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { throw new BoxRuntimeException( "Cannot evaluate a * expression" ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java index 05ea16c52..465da521f 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; /** * Abstract Node class representing SQL boolean @@ -68,14 +68,14 @@ public boolean isLiteral() { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return true; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java index 1840dcf81..23be9ecb5 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; /** * Abstract Node class representing SQL null @@ -48,7 +48,7 @@ public boolean isLiteral() { /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return null; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java index e10da0955..aadd6ad66 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java @@ -22,7 +22,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.Query; import ortus.boxlang.runtime.types.QueryColumnType; @@ -71,21 +71,21 @@ public boolean isLiteral() { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return true; } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return QueryColumnType.DOUBLE; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java index b1ae96ef4..d93fa70f6 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.QueryColumnType; /** @@ -65,14 +65,14 @@ public boolean isLiteral() { /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return QueryColumnType.VARCHAR; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java index 5fe76c3eb..a4755f3ec 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java @@ -21,7 +21,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.operators.Compare; /** @@ -120,14 +120,14 @@ public void setNot( boolean not ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return true; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Object leftValue = left.evaluate( QoQExec, intersection ); Object rightValue = right.evaluate( QoQExec, intersection ); Object expressionValue = expression.evaluate( QoQExec, intersection ); diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java index 8467248c7..2b8ce78fc 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java @@ -24,7 +24,7 @@ import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.LikeOperation; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.operators.Concat; import ortus.boxlang.runtime.operators.EqualsEquals; @@ -150,14 +150,14 @@ public void setEscape( SQLExpression escape ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return booleanOperators.contains( operator ); } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { // If this is a boolean operation, then we're a bit if ( isBoolean( QoQExec ) ) { return QueryColumnType.BIT; @@ -183,14 +183,14 @@ public QueryColumnType getType( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return getType( QoQExec ) == QueryColumnType.DOUBLE; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Object leftValue; Object rightValue; Double leftNum; @@ -294,7 +294,7 @@ public Object evaluate( QoQExecution QoQExec, int[] intersection ) { /** * Implement LIKE so we can reuse for NOT LIKE */ - private boolean doLike( QoQExecution QoQExec, int[] intersection ) { + private boolean doLike( QoQSelectExecution QoQExec, int[] intersection ) { String leftValueStr = StringCaster.cast( left.evaluate( QoQExec, intersection ) ); String rightValueStr = StringCaster.cast( right.evaluate( QoQExec, intersection ) ); String escapeValue = null; @@ -309,7 +309,7 @@ private boolean doLike( QoQExecution QoQExec, int[] intersection ) { * * @return true if the left and right operands are boolean expressions or bit columns */ - private void ensureBooleanOperands( QoQExecution QoQExec ) { + private void ensureBooleanOperands( QoQSelectExecution QoQExec ) { // These checks may or may not work. If we can't get away with this, then we can boolean cast the values // but SQL doesn't really have the same concept of truthiness and mostly expects to always get booleans from boolean columns or boolean expressions if ( !left.isBoolean( QoQExec ) ) { @@ -323,7 +323,7 @@ private void ensureBooleanOperands( QoQExecution QoQExec ) { /** * Reusable helper method to ensure that the left and right operands are numeric expressions or numeric columns */ - private void ensureNumericOperands( QoQExecution QoQExec ) { + private void ensureNumericOperands( QoQSelectExecution QoQExec ) { if ( !left.isNumeric( QoQExec ) ) { throw new BoxRuntimeException( "Left side of a math [" + operator.getSymbol() + "] operation must be a numeric expression or numeric column" ); } @@ -341,7 +341,7 @@ private void ensureNumericOperands( QoQExecution QoQExec ) { * * @return */ - private double evalAsNumber( SQLExpression expression, QoQExecution QoQExec, int[] intersection ) { + private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { return ( ( Number ) expression.evaluate( QoQExec, intersection ) ).doubleValue(); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java index 03197ad18..463d12a09 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java @@ -22,7 +22,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.operators.EqualsEquals; /** @@ -102,14 +102,14 @@ public void setNot( boolean not ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return true; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Object value = expression.evaluate( QoQExec, intersection ); for ( SQLExpression v : values ) { if ( EqualsEquals.invoke( value, v.evaluate( QoQExec, intersection ), true ) ) { diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java index 35295e08d..efbc7f223 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java @@ -22,7 +22,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -90,14 +90,14 @@ public void setOperator( SQLUnaryOperator operator ) { * * @return true if the expression evaluates to a boolean value */ - public boolean isBoolean( QoQExecution QoQExec ) { + public boolean isBoolean( QoQSelectExecution QoQExec ) { return booleanOperators.contains( operator ); } /** * What type does this expression evaluate to */ - public QueryColumnType getType( QoQExecution QoQExec ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { // If this is a boolean operation, then we're a bit if ( isBoolean( QoQExec ) ) { return QueryColumnType.BIT; @@ -115,14 +115,14 @@ public QueryColumnType getType( QoQExecution QoQExec ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( QoQExecution QoQExec ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { return getType( QoQExec ) == QueryColumnType.DOUBLE; } /** * Evaluate the expression */ - public Object evaluate( QoQExecution QoQExec, int[] intersection ) { + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { // Implement each unary operator switch ( operator ) { case ISNOTNULL : @@ -149,7 +149,7 @@ public Object evaluate( QoQExecution QoQExec, int[] intersection ) { * * @return true if the left and right operands are boolean expressions or bit columns */ - private void ensureBooleanOperand( QoQExecution QoQExec ) { + private void ensureBooleanOperand( QoQSelectExecution QoQExec ) { // These checks may or may not work. If we can't get away with this, then we can boolean cast the values // but SQL doesn't really have the same concept of truthiness and mostly expects to always get booleans from boolean columns or boolean expressions if ( !expression.isBoolean( QoQExec ) ) { @@ -160,7 +160,7 @@ private void ensureBooleanOperand( QoQExecution QoQExec ) { /** * Reusable helper method to ensure that the left and right operands are numeric expressions or numeric columns */ - private void ensureNumericOperand( QoQExecution QoQExec ) { + private void ensureNumericOperand( QoQSelectExecution QoQExec ) { if ( !expression.isNumeric( QoQExec ) ) { throw new BoxRuntimeException( "Unary operation [" + operator.getSymbol() + "] must be a numeric expression or numeric column" ); } @@ -175,7 +175,7 @@ private void ensureNumericOperand( QoQExecution QoQExec ) { * * @return */ - private double evalAsNumber( SQLExpression expression, QoQExecution QoQExec, int[] intersection ) { + private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { return ( ( Number ) expression.evaluate( QoQExec, intersection ) ).doubleValue(); } diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index c79a2599f..b17cfed0d 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -13,6 +13,10 @@ import ortus.boxlang.compiler.ast.sql.select.SQLSelect; import ortus.boxlang.compiler.ast.sql.select.SQLSelectStatement; import ortus.boxlang.compiler.ast.sql.select.SQLTable; +import ortus.boxlang.compiler.ast.sql.select.SQLTableSubQuery; +import ortus.boxlang.compiler.ast.sql.select.SQLTableVariable; +import ortus.boxlang.compiler.ast.sql.select.SQLUnion; +import ortus.boxlang.compiler.ast.sql.select.SQLUnionType; import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; import ortus.boxlang.compiler.ast.sql.select.expression.SQLCountFunction; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; @@ -42,7 +46,9 @@ import ortus.boxlang.parser.antlr.SQLGrammar.Select_stmtContext; import ortus.boxlang.parser.antlr.SQLGrammar.Sql_stmtContext; import ortus.boxlang.parser.antlr.SQLGrammar.Sql_stmt_listContext; +import ortus.boxlang.parser.antlr.SQLGrammar.SubqueryContext; import ortus.boxlang.parser.antlr.SQLGrammar.TableContext; +import ortus.boxlang.parser.antlr.SQLGrammar.Table_or_subqueryContext; import ortus.boxlang.parser.antlr.SQLGrammarBaseVisitor; import ortus.boxlang.runtime.scopes.Key; @@ -124,7 +130,7 @@ public BoxNode visitSelect_stmt( Select_stmtContext ctx ) { var src = tools.getSourceText( ctx ); SQLSelect select = null; - List unions = null; + List unions = null; List orderBys = null; SQLNumberLiteral limit = null; @@ -141,7 +147,27 @@ public BoxNode visitSelect_stmt( Select_stmtContext ctx ) { limit = NUMERIC_LITERAL( ctx.limit_stmt().NUMERIC_LITERAL() ); } - // TODO: handle unions + if ( ctx.union() != null && !ctx.union().isEmpty() ) { + int numCols = select.getResultColumns().size(); + unions = new ArrayList(); + int numSelect = 2; + for ( var unionCtx : ctx.union() ) { + SQLUnionType unionType = unionCtx.ALL_() != null ? SQLUnionType.ALL : SQLUnionType.DISTINCT; + if ( unionType == SQLUnionType.ALL && unionCtx.DISTINCT_() != null ) { + tools.reportError( "Cannot have both ALL and DISTINCT in a UNION", tools.getPosition( unionCtx ) ); + } + var unionSelect = ( SQLSelect ) visit( unionCtx.select_core() ); + if ( unionSelect.getResultColumns().size() != numCols ) { + tools.reportError( + "All SELECT statements in a UNION must have the same number of columns. Select number " + numSelect + " has " + + unionSelect.getResultColumns().size() + " column(s) but the original select has " + numCols + " column(s).", + tools.getPosition( unionCtx ) ); + } + unions.add( new SQLUnion( unionSelect, unionType, tools.getPosition( unionCtx ), tools.getSourceText( unionCtx ) ) ); + numSelect++; + } + + } return new SQLSelectStatement( select, unions, orderBys, limit, pos, src ); } @@ -156,34 +182,34 @@ public BoxNode visitSelect_stmt( Select_stmtContext ctx ) { */ @Override public SQLSelect visitSelect_core( Select_coreContext ctx ) { - var pos = tools.getPosition( ctx ); - var src = tools.getSourceText( ctx ); - - boolean distinct = ctx.DISTINCT_() != null; - SQLNumberLiteral limit = null; - List resultColumns = null; - SQLTable table = null; - List joins = null; - SQLExpression where = null; - List groupBys = null; - SQLExpression having = null; - - TableContext firstTable; - if ( !ctx.table().isEmpty() ) { - firstTable = ctx.table().get( 0 ); + var pos = tools.getPosition( ctx ); + var src = tools.getSourceText( ctx ); + + boolean distinct = ctx.DISTINCT_() != null; + SQLNumberLiteral limit = null; + List resultColumns = null; + SQLTable table = null; + List joins = null; + SQLExpression where = null; + List groupBys = null; + SQLExpression having = null; + + Table_or_subqueryContext firstTable; + if ( !ctx.table_or_subquery().isEmpty() ) { + firstTable = ctx.table_or_subquery().get( 0 ); table = ( SQLTable ) visit( firstTable ); - if ( ctx.table().size() > 1 ) { + if ( ctx.table_or_subquery().size() > 1 ) { // from table1, table2 is treated as a join with no `on` clause joins = new ArrayList(); - for ( int i = 1; i < ctx.table().size(); i++ ) { - var tableCtx = ctx.table().get( i ); + for ( int i = 1; i < ctx.table_or_subquery().size(); i++ ) { + var tableCtx = ctx.table_or_subquery().get( i ); SQLTable joinTable = ( SQLTable ) visit( tableCtx ); joins.add( new SQLJoin( SQLJoinType.FULL, joinTable, null, tools.getPosition( tableCtx ), tools.getSourceText( tableCtx ) ) ); } } } else if ( ctx.join_clause() != null ) { - firstTable = ctx.join_clause().table(); + firstTable = ctx.join_clause().table_or_subquery(); table = ( SQLTable ) visit( firstTable ); joins = buildJoins( ctx.join_clause(), table ); } @@ -230,7 +256,7 @@ public List buildJoins( SQLGrammar.Join_clauseContext ctx, SQLTable tab for ( var joinCtx : ctx.join() ) { var pos = tools.getPosition( joinCtx ); var src = tools.getSourceText( joinCtx ); - var joinTable = ( SQLTable ) visit( joinCtx.table() ); + var joinTable = ( SQLTable ) visit( joinCtx.table_or_subquery() ); var typeCtx = joinCtx.join_operator(); boolean hasOn = joinCtx.join_constraint() != null; String joinText = tools.getSourceText( typeCtx ); @@ -277,7 +303,24 @@ public List buildJoins( SQLGrammar.Join_clauseContext ctx, SQLTable tab * @return the AST node representing the class or interface */ @Override - public SQLTable visitTable( TableContext ctx ) { + public SQLTable visitTable_or_subquery( Table_or_subqueryContext ctx ) { + if ( ctx.table() != null ) { + return visitTable( ctx.table() ); + } else { + return visitSubquery( ctx.subquery() ); + } + } + + /** + * Visit the class or interface context to generate the AST node for the + * top level node + * + * @param ctx the parse tree + * + * @return the AST node representing the class or interface + */ + @Override + public SQLTableVariable visitTable( TableContext ctx ) { var pos = tools.getPosition( ctx ); var src = tools.getSourceText( ctx ); String schema = null; @@ -292,7 +335,24 @@ public SQLTable visitTable( TableContext ctx ) { alias = ctx.table_alias().getText(); } - return new SQLTable( schema, name, alias, tableIndex++, pos, src ); + return new SQLTableVariable( schema, name, alias, tableIndex++, pos, src ); + } + + /** + * Visit the class or interface context to generate the AST node for the + * top level node + * + * @param ctx the parse tree + * + * @return the AST node representing the class or interface + */ + @Override + public SQLTableSubQuery visitSubquery( SubqueryContext ctx ) { + var pos = tools.getPosition( ctx ); + var src = tools.getSourceText( ctx ); + SQLSelectStatement select = ( SQLSelectStatement ) visit( ctx.select_stmt() ); + + return new SQLTableSubQuery( select, ctx.table_alias().getText(), tableIndex++, pos, src ); } /** diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java index 6d5b60a5d..24ef8edb5 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java @@ -21,7 +21,7 @@ /** * I am the abstract class for QoQ function definitions */ -public abstract class QoQAggregateFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, QoQExecution, Object> { +public abstract class QoQAggregateFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, QoQSelectExecution, Object> { public boolean isAggregate() { return true; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index a38326b25..603d7dfa0 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -15,12 +15,8 @@ package ortus.boxlang.runtime.jdbc.qoq; import java.sql.SQLException; -import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Stream; import ortus.boxlang.compiler.ast.sql.SQLNode; @@ -29,10 +25,11 @@ import ortus.boxlang.compiler.ast.sql.select.SQLSelect; import ortus.boxlang.compiler.ast.sql.select.SQLSelectStatement; import ortus.boxlang.compiler.ast.sql.select.SQLTable; -import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; +import ortus.boxlang.compiler.ast.sql.select.SQLTableSubQuery; +import ortus.boxlang.compiler.ast.sql.select.SQLTableVariable; +import ortus.boxlang.compiler.ast.sql.select.SQLUnion; +import ortus.boxlang.compiler.ast.sql.select.SQLUnionType; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; -import ortus.boxlang.compiler.ast.sql.select.expression.SQLStarExpression; -import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNumberLiteral; import ortus.boxlang.compiler.parser.ParsingResult; import ortus.boxlang.compiler.parser.SQLParser; import ortus.boxlang.runtime.BoxRuntime; @@ -82,62 +79,107 @@ public static SQLNode parseSQL( String sql ) { return ( SQLNode ) result.getRoot(); } - public static Query executeSelect( IBoxContext context, SQLSelectStatement selectStatement, QoQStatement statement ) throws SQLException { - Map tableLookup = new LinkedHashMap(); - boolean hasTable = selectStatement.getSelect().getTable() != null; - Query source = null; + public static Query executeSelectStatement( IBoxContext context, SQLSelectStatement selectStatement, QoQStatement statement ) { - // TODO: Process all joins - if ( hasTable ) { - String tableVarName = selectStatement.getSelect().getTable().getVariableName(); - source = getSourceQuery( context, tableVarName ); - // TODO: ensure tables are added in the order of their index, which represents the encounter-order in the SQL - tableLookup.put( selectStatement.getSelect().getTable(), source ); + QoQSelectStatementExecution QoQStmtExec = QoQSelectStatementExecution.of( + selectStatement, + statement instanceof QoQPreparedStatement qp ? qp.getParameters() : null, + statement ); + + Query target = executeSelect( context, selectStatement.getSelect(), statement, QoQStmtExec, true ); + + if ( selectStatement.getUnions() != null ) { + for ( SQLUnion union : selectStatement.getUnions() ) { + Query unionQuery = executeSelect( context, union.getSelect(), statement, QoQStmtExec, false ); + if ( union.getType() == SQLUnionType.ALL ) { + unionAll( target, unionQuery ); + } else { + // distinct + unionDistinct( target, unionQuery ); + } + } } - // Register joins - if ( selectStatement.getSelect().getJoins() != null ) { - for ( SQLJoin thisJoin : selectStatement.getSelect().getJoins() ) { - SQLTable table = thisJoin.getTable(); - String tableVarName = table.getVariableName(); - tableLookup.put( table, getSourceQuery( context, tableVarName ) ); + // TODO: Implement a sort that doesn't turn the query into a list of structs and back again + if ( QoQStmtExec.getOrderByColumns() != null ) { + target.sort( ( row1, row2 ) -> { + var orderBys = QoQStmtExec.getOrderByColumns(); + for ( var orderBy : orderBys ) { + var name = orderBy.name; + int result = Compare.invoke( row1.get( name ), row2.get( name ) ); + if ( result != 0 ) { + return orderBy.ascending ? result : -result; + } + } + return 0; + } ); + // These were just here for sorting. Nuke them now. + if ( QoQStmtExec.getAdditionalColumns() != null ) { + for ( Key key : QoQStmtExec.getAdditionalColumns() ) { + target.deleteColumn( key ); + } } - hasTable = true; } - // This holds the AST and the runtime values for the query - QoQExecution QoQExec = QoQExecution.of( - selectStatement, - tableLookup, - statement instanceof QoQPreparedStatement qp ? qp.getParameters() : null - ); + // This is the maxRows in the query options. It takes priority. + Long overallSelectLimit; + try { + overallSelectLimit = statement.getLargeMaxRows(); + } catch ( SQLException e ) { + throw new DatabaseException( "Error getting max rows from statement", e ); + } + // If that wasn't set, use the limit clause AFTER the order by (which could apply at the end of a union) + if ( overallSelectLimit == -1 ) { + overallSelectLimit = selectStatement.getLimitValue(); + } - Stream intersections = null; - if ( hasTable ) { - intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); + // If we have a limit for the final select, apply it here. + if ( overallSelectLimit > -1 ) { + target.truncate( overallSelectLimit ); } + return target; + } - Map resultColumns = calculateResultColumns( QoQExec ); - calculateOrderBys( QoQExec ); + public static Query executeSelect( IBoxContext context, SQLSelect select, QoQStatement statement, QoQSelectStatementExecution QoQStmtExec, + boolean firstSelect ) { + boolean canEarlyLimit = QoQStmtExec.getSelectStatement().getOrderBys() == null; + Map tableLookup = new LinkedHashMap(); + // This boolean expression will be used to filter the records we keep + SQLExpression where = select.getWhere(); + boolean hasTable = select.getTable() != null; + Long thisSelectLimit = select.getLimitValue(); - Query target = buildTargetQuery( QoQExec ); + if ( hasTable ) { + // Tables are added in the order of their index, which represents the encounter-order in the SQL + tableLookup.put( select.getTable(), getSourceQuery( context, QoQStmtExec, select.getTable() ) ); + + // Register joins + if ( select.getJoins() != null ) { + for ( SQLJoin thisJoin : select.getJoins() ) { + SQLTable table = thisJoin.getTable(); + tableLookup.put( table, getSourceQuery( context, QoQStmtExec, table ) ); + } + } + } - // print out arrays - /* - * intersections.forEach( i -> System.out.println( Arrays.toString( i ) ) ); - * if ( true ) - * return target; - */ - // Process one select - // TODO: refactor this out - SQLSelect select = selectStatement.getSelect(); + // This holds the AST and the runtime values for the query + QoQSelectExecution QoQExec = QoQStmtExec.newQoQSelectExecution( + select, + tableLookup + ); + // Calculate the result columns for this select + Map resultColumns = QoQExec.calculateResultColumns( firstSelect ); - Long thisSelectLimit = select.getLimitValue(); - // This boolean expression will be used to filter the records we keep - SQLExpression where = select.getWhere(); - boolean canEarlyLimit = selectStatement.getOrderBys() == null; + // If this is the first select, (not a union) calculate order bys, which may modify the result columns + if ( firstSelect ) { + QoQExec.calculateOrderBys(); + } + + // Create empty query object to hold result + Query target = buildTargetQuery( QoQExec ); - // Just selecting out literal values + // If there are no tables, and we are just selecting out literal values, we can just add the row and return + // This code path ignores the where clause and top/limit. While technically vaid, it is not a common use case. if ( !hasTable ) { Object[] values = new Object[ resultColumns.size() ]; for ( Key key : resultColumns.keySet() ) { @@ -149,105 +191,62 @@ public static Query executeSelect( IBoxContext context, SQLSelectStatement selec return target; } + // We have one or more tables, so build our stream of intersections, processing our joins as needed + Stream intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); + if ( select.hasAggregateResult() ) { } - // enforce top/limit for this select. This would be a "top N" clause in the select or a "limit N" clause BEFORE the order by, which + // Enforce top/limit for this select. This would be a "top N" clause in the select or a "limit N" clause BEFORE the order by, which // could exist or all selects in a union. if ( canEarlyLimit && thisSelectLimit > -1 ) { intersections = intersections.limit( thisSelectLimit ); + } + // If we have a where clause, add it as a filter to the stream + if ( where != null ) { + intersections = intersections.filter( intersection -> ( Boolean ) where.evaluate( QoQExec, intersection ) ); } - // 1-based index! + // Process/create the rows for the final query. intersections.forEach( intersection -> { // System.out.println( Arrays.toString( intersection ) ); - // Evaluate the where expression - if ( where == null || ( Boolean ) where.evaluate( QoQExec, intersection ) ) { - Object[] values = new Object[ resultColumns.size() ]; - int colPos = 0; - for ( Key key : resultColumns.keySet() ) { - SQLResultColumn resultColumn = resultColumns.get( key ).resultColumn; - Object value = resultColumn.getExpression().evaluate( QoQExec, intersection ); - values[ colPos++ ] = value; - } - target.addRow( values ); + Object[] values = new Object[ resultColumns.size() ]; + int colPos = 0; + // Build up row data as native array + for ( Key key : resultColumns.keySet() ) { + SQLResultColumn resultColumn = resultColumns.get( key ).resultColumn; + Object value = resultColumn.getExpression().evaluate( QoQExec, intersection ); + values[ colPos++ ] = value; } + target.addRow( values ); } ); - // TODO: Implement a sort that doesn't turn the query into a list of structs and back again - if ( QoQExec.getOrderByColumns() != null ) { - target.sort( ( row1, row2 ) -> { - var orderBys = QoQExec.getOrderByColumns(); - for ( var orderBy : orderBys ) { - var name = orderBy.name; - int result = Compare.invoke( row1.get( name ), row2.get( name ) ); - if ( result != 0 ) { - return orderBy.ascending ? result : -result; - } - } - return 0; - } ); - // These were just here for sorting. Nuke them now. - if ( QoQExec.getAdditionalColumns() != null ) { - for ( Key key : QoQExec.getAdditionalColumns() ) { - target.deleteColumn( key ); - } - } - } - - // This is the maxRows in the query options. It takes priority. - Long overallSelectLimit = statement.getLargeMaxRows(); - // If that wasn't set, use the limit clause AFTER the order by (which could apply at the end of a union) - if ( overallSelectLimit == -1 ) { - overallSelectLimit = selectStatement.getLimitValue(); - } - - // If we have a limit for the final select, apply it here. - if ( overallSelectLimit > -1 ) { - target.truncate( overallSelectLimit ); - } return target; } - private static void calculateOrderBys( QoQExecution qoQExec ) { - SQLSelectStatement selectStatement = qoQExec.select; - if ( selectStatement.getOrderBys() == null ) { - return; + /** + * Union two queries together, keeping all rows + * + * @param target the target query + * @param unionQuery the query to union + */ + private static void unionAll( Query target, Query unionQuery ) { + for ( int i = 0; i < unionQuery.size(); i++ ) { + target.addRow( unionQuery.getRow( i ) ); } - Set additionalColumns = new LinkedHashSet(); - Map resultColumns = qoQExec.getResultColumns(); - List orderByColumns = new ArrayList(); - int additionalCounter = 1; - for ( var orderBy : selectStatement.getOrderBys() ) { - SQLExpression expr = orderBy.getExpression(); - if ( expr instanceof SQLColumn column ) { - var match = resultColumns.entrySet().stream().filter( rc -> column.getName().equals( rc.getKey() ) ).findFirst(); - if ( match.isPresent() ) { - orderByColumns.add( NameAndDirection.of( match.get().getKey(), orderBy.isAscending() ) ); - continue; - } - } else if ( expr instanceof SQLNumberLiteral num ) { - // This is a number literal, which is a 1-based index into the result set - int index = num.getValue().intValue(); - if ( index < 1 || index > resultColumns.size() ) { - throw new DatabaseException( "The column index [" + index + "] in the order by clause is out of range." ); - } - orderByColumns.add( NameAndDirection.of( resultColumns.keySet().toArray( new Key[ 0 ] )[ index - 1 ], orderBy.isAscending() ) ); - continue; - } - // TODO: Figure out if this exact expression is already in the result set and use that - // To do this, we need something like toString() implemented to compare two expressions for equivalence - Key newName = Key.of( "__order_by_column_" + additionalCounter++ ); - resultColumns.put( newName, - TypedResultColumn.of( QueryColumnType.OBJECT, new SQLResultColumn( expr, newName.getName(), resultColumns.size() + 1, null, null ) ) ); - orderByColumns.add( NameAndDirection.of( newName, orderBy.isAscending() ) ); - additionalColumns.add( newName ); + } - } - qoQExec.setOrderByColumns( orderByColumns ); - qoQExec.setAdditionalColumns( additionalColumns ); + /** + * Union two queries together, keeping only distinct rows + * + * @param target the target query + * @param unionQuery the query to union + */ + private static void unionDistinct( Query target, Query unionQuery ) { + // TODO: IMPLEMENT! + unionAll( target, unionQuery ); } /** @@ -257,7 +256,7 @@ private static void calculateOrderBys( QoQExecution qoQExec ) { * * @return the target query */ - private static Query buildTargetQuery( QoQExecution QoQExec ) { + private static Query buildTargetQuery( QoQSelectExecution QoQExec ) { Map resultColumns = QoQExec.getResultColumns(); Query target = new Query(); for ( Key key : resultColumns.keySet() ) { @@ -266,57 +265,22 @@ private static Query buildTargetQuery( QoQExecution QoQExec ) { return target; } - private static Map calculateResultColumns( QoQExecution QoQExec ) { - Map resultColumns = new LinkedHashMap(); - for ( SQLResultColumn resultColumn : QoQExec.select.getSelect().getResultColumns() ) { - // For *, expand all columns in the query - if ( resultColumn.isStarExpression() ) { - // The same table joined more than once will still have separate SQLTable instances in the AST. - // If we have a specific alias such as t.* this will still match since the correct SQLTable reference will be associated with the result column - SQLTable starTable = ( ( SQLStarExpression ) resultColumn.getExpression() ).getTable(); - Key tableName = starTable == null ? null : starTable.getName(); - - var matchingTables = QoQExec.tableLookup.keySet().stream().filter( t -> starTable == null || starTable == t ) - .toList(); - - if ( matchingTables.isEmpty() ) { - throw new DatabaseException( - "The table alias [" + tableName + "] in the result column [" + resultColumn.getSourceText() + "] is does not match a table." ); - } - matchingTables.stream().forEach( t -> { - var thisTable = QoQExec.tableLookup.get( t ); - for ( Key key : thisTable.getColumns().keySet() ) { - resultColumns.put( key, - TypedResultColumn.of( - thisTable.getColumns().get( key ).getType(), - new SQLResultColumn( - new SQLColumn( t, key.getName(), null, null ), - null, - resultColumns.size() + 1, - null, - null - ) - ) - ); - } - } ); - // Non-star columns are named after the column, or given a column_0, column_1, etc name + private static Query getSourceQuery( IBoxContext context, QoQSelectStatementExecution QoQStmtExec, SQLTable table ) { + if ( table instanceof SQLTableVariable tableVar ) { + String tableVarName = tableVar.getVariableName(); + Object oSource = ExpressionInterpreter.getVariable( context, tableVarName, false ); + if ( oSource instanceof Query qSource ) { + return qSource; + } else if ( oSource == null ) { + throw new DatabaseException( "The QoQ table name [" + tableVarName + "] cannot be found as a variable." ); } else { - resultColumns.put( resultColumn.getResultColumnName(), - TypedResultColumn.of( resultColumn.getExpression().getType( QoQExec ), resultColumn ) ); + throw new DatabaseException( + "The QoQ table name [" + tableVarName + "] is not of type query, but instead is [" + oSource.getClass().getName() + "]" ); } + } else if ( table instanceof SQLTableSubQuery tableSub ) { + return executeSelectStatement( context, tableSub.getSelectStatement(), QoQStmtExec.getJDBCStatement() ); } - QoQExec.setResultColumns( resultColumns ); - return resultColumns; - } - - private static Query getSourceQuery( IBoxContext context, String tableVarName ) { - Object oSource = ExpressionInterpreter.getVariable( context, tableVarName, false ); - if ( oSource instanceof Query qSource ) { - return qSource; - } else { - throw new DatabaseException( "The QoQ table name [" + tableVarName + "] cannot be found as a variable." ); - } + throw new DatabaseException( "Unknown table type [" + table.getClass().getName() + "]" ); } public record TypedResultColumn( QueryColumnType type, SQLResultColumn resultColumn ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index fcbcd0c01..04cc24886 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -92,7 +92,7 @@ public static void register( Key name, java.util.function.Function, functions.put( name, QoQFunction.of( function, returnType, requiredParams ) ); } - public static void registerAggregate( Key name, java.util.function.BiFunction, QoQExecution, Object> function, + public static void registerAggregate( Key name, java.util.function.BiFunction, QoQSelectExecution, Object> function, QueryColumnType returnType, int requiredParams ) { functions.put( name, QoQFunction.ofAggregate( function, returnType, requiredParams ) ); @@ -119,7 +119,7 @@ public static QoQFunction getFunction( Key name ) { public record QoQFunction( java.util.function.Function, Object> callable, - java.util.function.BiFunction, QoQExecution, Object> aggregateCallable, + java.util.function.BiFunction, QoQSelectExecution, Object> aggregateCallable, QueryColumnType returnType, int requiredParams ) { @@ -127,7 +127,7 @@ static QoQFunction of( java.util.function.Function, Object> callabl return new QoQFunction( callable, null, returnType, requiredParams ); } - static QoQFunction ofAggregate( java.util.function.BiFunction, QoQExecution, Object> callable, QueryColumnType returnType, + static QoQFunction ofAggregate( java.util.function.BiFunction, QoQSelectExecution, Object> callable, QueryColumnType returnType, int requiredParams ) { return new QoQFunction( null, callable, returnType, requiredParams ); } @@ -136,7 +136,7 @@ public Object invoke( List arguments ) { return callable.apply( arguments ); } - public Object invokeAggregate( List arguments, QoQExecution QoQExec ) { + public Object invokeAggregate( List arguments, QoQSelectExecution QoQExec ) { return aggregateCallable.apply( arguments, QoQExec ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java index 515fb7883..3a06d249c 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java @@ -37,9 +37,9 @@ public class QoQIntersectionGenerator { /** * Create all possible intersections of the tables as a stream of int arrays */ - public static Stream createIntersectionStream( QoQExecution QoQExec ) { + public static Stream createIntersectionStream( QoQSelectExecution QoQExec ) { Map tableLookup = QoQExec.tableLookup; - Query firstTable = tableLookup.get( QoQExec.select.getSelect().getTable() ); + Query firstTable = tableLookup.get( QoQExec.getSelect().getTable() ); // This is just an estimation of the total number of combinations // Streams make it hard to get an exact count without collecting the stream, so we'll just guess int totalCombinations = firstTable.size(); @@ -49,8 +49,8 @@ public static Stream createIntersectionStream( QoQExecution QoQExec ) { .mapToObj( i -> new int[] { i } ); // If we have one or more joins, we need to create a stream of all possible intersections - if ( QoQExec.select.getSelect().getJoins() != null ) { - for ( SQLJoin thisJoin : QoQExec.select.getSelect().getJoins() ) { + if ( QoQExec.getSelect().getJoins() != null ) { + for ( SQLJoin thisJoin : QoQExec.getSelect().getJoins() ) { var joinType = thisJoin.getType(); var joinTable = tableLookup.get( thisJoin.getTable() ); var joinOn = thisJoin.getOn(); @@ -78,7 +78,7 @@ public static Stream createIntersectionStream( QoQExecution QoQExec ) { return theStream; } - private static Stream handleCrossOrInnerJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + private static Stream handleCrossOrInnerJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { theStream = theStream.flatMap( i -> IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); newIntersection[ i.length ] = j; @@ -90,7 +90,7 @@ private static Stream handleCrossOrInnerJoin( Stream theStream, Qu return theStream; } - private static Stream handleLeftJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + private static Stream handleLeftJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { return theStream.flatMap( i -> { Stream newStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); @@ -107,7 +107,7 @@ private static Stream handleLeftJoin( Stream theStream, Query join } ); } - private static Stream handleRightJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + private static Stream handleRightJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); return rightStream.flatMap( j -> { @@ -129,7 +129,7 @@ private static Stream handleRightJoin( Stream theStream, Query joi } ); } - private static Stream handleFullOuterJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQExecution QoQExec ) { + private static Stream handleFullOuterJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQPreparedStatement.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQPreparedStatement.java index c47d44285..00242ca9f 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQPreparedStatement.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQPreparedStatement.java @@ -215,7 +215,7 @@ public boolean execute() throws SQLException { SQLSelectStatement select = ( SQLSelectStatement ) QoQExecutionService.parseSQL( sql ); // execute the query - result = QoQExecutionService.executeSelect( context, select, this ); + result = QoQExecutionService.executeSelectStatement( context, select, this ); return true; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java new file mode 100644 index 000000000..6efd7cf46 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -0,0 +1,186 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import ortus.boxlang.compiler.ast.sql.select.SQLResultColumn; +import ortus.boxlang.compiler.ast.sql.select.SQLSelect; +import ortus.boxlang.compiler.ast.sql.select.SQLSelectStatement; +import ortus.boxlang.compiler.ast.sql.select.SQLTable; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLStarExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNumberLiteral; +import ortus.boxlang.runtime.jdbc.qoq.QoQExecutionService.NameAndDirection; +import ortus.boxlang.runtime.jdbc.qoq.QoQExecutionService.TypedResultColumn; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.QueryColumnType; +import ortus.boxlang.runtime.types.exceptions.DatabaseException; + +/** + * A wrapper class to hold together both the SQL AST being executed as well as the runtime values for a given execution of the query + */ +public class QoQSelectExecution { + + public SQLSelect select; + public Map resultColumns = null; + public Map tableLookup; + public QoQSelectStatementExecution selectStatementExecution; + + /** + * Constructor + * + * @param select + * @param tableLookup + * @param params + * + * @return + */ + private QoQSelectExecution( SQLSelect select, Map tableLookup ) { + this.select = select; + this.tableLookup = tableLookup; + } + + public static QoQSelectExecution of( SQLSelect select, Map tableLookup ) { + return new QoQSelectExecution( select, tableLookup ); + } + + public SQLSelect getSelect() { + return select; + } + + public Map getTableLookup() { + return tableLookup; + } + + public Map getResultColumns() { + return resultColumns; + } + + public void setResultColumns( Map resultColumns ) { + this.resultColumns = resultColumns; + } + + public QoQSelectStatementExecution getSelectStatementExecution() { + return selectStatementExecution; + } + + public void setQoQSelectStatementExecution( QoQSelectStatementExecution selectStatementExecution ) { + this.selectStatementExecution = selectStatementExecution; + } + + public Map calculateResultColumns( boolean firstSelect ) { + Map resultColumns = new LinkedHashMap(); + for ( SQLResultColumn resultColumn : getSelect().getResultColumns() ) { + // For *, expand all columns in the query + if ( resultColumn.isStarExpression() ) { + // The same table joined more than once will still have separate SQLTable instances in the AST. + // If we have a specific alias such as t.* this will still match since the correct SQLTable reference will be associated with the result column + SQLTable starTable = ( ( SQLStarExpression ) resultColumn.getExpression() ).getTable(); + + var matchingTables = tableLookup.keySet().stream().filter( t -> starTable == null || starTable == t ) + .toList(); + + if ( matchingTables.isEmpty() ) { + throw new DatabaseException( + "The table reference in the result column [" + resultColumn.getSourceText() + "] does not match a table." ); + } + matchingTables.stream().forEach( t -> { + var thisTable = tableLookup.get( t ); + for ( Key key : thisTable.getColumns().keySet() ) { + resultColumns.put( key, + TypedResultColumn.of( + thisTable.getColumns().get( key ).getType(), + new SQLResultColumn( + new SQLColumn( t, key.getName(), null, null ), + null, + resultColumns.size() + 1, + null, + null + ) + ) + ); + } + } ); + // Non-star columns are named after the column, or given a column_0, column_1, etc name + } else { + resultColumns.put( resultColumn.getResultColumnName(), + TypedResultColumn.of( resultColumn.getExpression().getType( this ), resultColumn ) ); + } + } + setResultColumns( resultColumns ); + // Given a union, the first result set names are used for all unioned selects as well + if ( firstSelect ) { + getSelectStatementExecution().setResultColumnNames( resultColumns.keySet() ); + } + return resultColumns; + } + + public void calculateOrderBys() { + var QoQStmtExec = selectStatementExecution; + SQLSelectStatement selectStatement = QoQStmtExec.selectStatement; + if ( selectStatement.getOrderBys() == null ) { + return; + } + boolean isUnion = selectStatement.getUnions() != null; + Set additionalColumns = new LinkedHashSet(); + Map resultColumns = getResultColumns(); + List orderByColumns = new ArrayList(); + int additionalCounter = 1; + int numOriginalResulColumns = resultColumns.size(); + for ( var orderBy : selectStatement.getOrderBys() ) { + SQLExpression expr = orderBy.getExpression(); + if ( expr instanceof SQLColumn column ) { + var match = resultColumns.entrySet().stream().filter( rc -> column.getName().equals( rc.getKey() ) ).findFirst(); + if ( match.isPresent() ) { + orderByColumns.add( NameAndDirection.of( match.get().getKey(), orderBy.isAscending() ) ); + continue; + } + } else if ( expr instanceof SQLNumberLiteral num ) { + // This is a number literal, which is a 1-based index into the result set + int index = num.getValue().intValue(); + if ( index < 1 || index > numOriginalResulColumns ) { + throw new DatabaseException( "The column index [" + index + "] in the order by clause is out of range as there are only " + + numOriginalResulColumns + " column(s)." ); + } + orderByColumns.add( NameAndDirection.of( resultColumns.keySet().toArray( new Key[ 0 ] )[ index - 1 ], orderBy.isAscending() ) ); + continue; + } + // TODO: This isn't quite right as a literal expression is technically OK in the order by of a union query, even though it's fairly useless. + // We need the query sort to be rewritten to eval expressions on the fly for that to work however. Not worth addressing at the moment. + if ( isUnion ) { + throw new DatabaseException( "The order by clause in a union query must reference a column by name that is in the select list or index." ); + } + // TODO: Figure out if this exact expression is already in the result set and use that + // To do this, we need something like toString() implemented to compare two expressions for equivalence + Key newName = Key.of( "__order_by_column_" + additionalCounter++ ); + resultColumns.put( newName, + TypedResultColumn.of( QueryColumnType.OBJECT, new SQLResultColumn( expr, newName.getName(), resultColumns.size() + 1, null, null ) ) ); + orderByColumns.add( NameAndDirection.of( newName, orderBy.isAscending() ) ); + additionalColumns.add( newName ); + + } + QoQStmtExec.setOrderByColumns( orderByColumns ); + QoQStmtExec.setAdditionalColumns( additionalColumns ); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java similarity index 51% rename from src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java rename to src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java index c7672e0f9..975f54b3d 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java @@ -14,14 +14,15 @@ */ package ortus.boxlang.runtime.jdbc.qoq; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import ortus.boxlang.compiler.ast.sql.select.SQLSelect; import ortus.boxlang.compiler.ast.sql.select.SQLSelectStatement; import ortus.boxlang.compiler.ast.sql.select.SQLTable; import ortus.boxlang.runtime.jdbc.qoq.QoQExecutionService.NameAndDirection; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecutionService.TypedResultColumn; import ortus.boxlang.runtime.jdbc.qoq.QoQPreparedStatement.ParamItem; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Query; @@ -29,14 +30,15 @@ /** * A wrapper class to hold together both the SQL AST being executed as well as the runtime values for a given execution of the query */ -public class QoQExecution { +public class QoQSelectStatementExecution { - public SQLSelectStatement select; - public Map resultColumns = null; - public Map tableLookup; - public List params; - public List orderByColumns = null; - public Set additionalColumns = null; + public SQLSelectStatement selectStatement; + public Set resultColumnNames = null; + public List params; + public List orderByColumns = null; + public Set additionalColumns = null; + public List selects = new ArrayList(); + QoQStatement JDBCStatement; /** * Constructor @@ -47,34 +49,30 @@ public class QoQExecution { * * @return */ - private QoQExecution( SQLSelectStatement select, Map tableLookup, List params ) { - this.select = select; - this.tableLookup = tableLookup; - this.params = params; + private QoQSelectStatementExecution( SQLSelectStatement selectStatement, List params, QoQStatement JDBCStatement ) { + this.selectStatement = selectStatement; + this.params = params; + this.JDBCStatement = JDBCStatement; } - public static QoQExecution of( SQLSelectStatement select, Map tableLookup, List params ) { - return new QoQExecution( select, tableLookup, params ); + public static QoQSelectStatementExecution of( SQLSelectStatement selectStatement, List params, QoQStatement JDBCStatement ) { + return new QoQSelectStatementExecution( selectStatement, params, JDBCStatement ); } - public SQLSelectStatement getSelect() { - return select; - } - - public Map getTableLookup() { - return tableLookup; + public SQLSelectStatement getSelectStatement() { + return selectStatement; } public List getParams() { return params; } - public Map getResultColumns() { - return resultColumns; + public Set getResultColumnName() { + return resultColumnNames; } - public void setResultColumns( Map resultColumns ) { - this.resultColumns = resultColumns; + public void setResultColumnNames( Set resultColumnNames ) { + this.resultColumnNames = resultColumnNames; } public List getOrderByColumns() { @@ -85,6 +83,10 @@ public void setOrderByColumns( List orderByColumns ) { this.orderByColumns = orderByColumns; } + public List getSelects() { + return selects; + } + public Set getAdditionalColumns() { return additionalColumns; } @@ -93,4 +95,20 @@ public void setAdditionalColumns( Set additionalColumns ) { this.additionalColumns = additionalColumns; } + public QoQSelectStatementExecution addSelect( QoQSelectExecution select ) { + select.setQoQSelectStatementExecution( this ); + selects.add( select ); + return this; + } + + public QoQSelectExecution newQoQSelectExecution( SQLSelect select, Map tableLookup ) { + var QoQExec = QoQSelectExecution.of( select, tableLookup ); + addSelect( QoQExec ); + return QoQExec; + } + + public QoQStatement getJDBCStatement() { + return JDBCStatement; + } + } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java index 63cd8c821..dd0b8079d 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java @@ -178,7 +178,7 @@ public boolean execute( String sql, int autoGeneratedKeys ) throws SQLException SQLSelectStatement select = ( SQLSelectStatement ) QoQExecutionService.parseSQL( sql ); // execute the query - result = QoQExecutionService.executeSelect( context, select, this ); + result = QoQExecutionService.executeSelectStatement( context, select, this ); return true; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java index afb06df79..e061c4a15 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java @@ -19,7 +19,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; -import ortus.boxlang.runtime.jdbc.qoq.QoQExecution; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -45,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args, QoQExecution QoQExec ) { + public Object apply( List args, QoQSelectExecution QoQExec ) { return ortus.boxlang.runtime.bifs.global.math.Atn._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 4ee6d134b..10d66f2c6 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -107,4 +107,43 @@ where d.name in ('IT','HR') context ); } + @Test + public void testRunQoQUnion() { + instance.executeSource( + """ + qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200],["Janitor",200]] ) + q = queryExecute( " + select name as col from qryDept + union all select 'bar' as sfd + union select 'foo' as col + order by col desc + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + } + + @Test + public void testSubquery() { + instance.executeSource( + """ + q = queryExecute( " + select col as brad from ( + select 'foo' as col + union select 'bar' + ) as t + order by brad asc + + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + } + } From 620db33c7d53089ef9a6220afd23ba415e2a9102 Mon Sep 17 00:00:00 2001 From: bdw429s Date: Sun, 15 Dec 2024 00:24:25 +0000 Subject: [PATCH 006/161] Apply cfformat changes --- .../ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java index 0cb4f2216..c29dcf833 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/SQLTableVariable.java @@ -70,7 +70,7 @@ public Key getName() { public void setName( String name ) { this.name = Key.of( name ); } - + public boolean isCalled( Key name ) { return this.name.equals( name ) || ( alias != null && alias.equals( name ) ); } From 145833aaa0ed40f48839dd0e1e2a689533da7923 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 14 Dec 2024 19:07:24 -0600 Subject: [PATCH 007/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 5 + .../operation/SQLInSubQueryOperation.java | 149 ++++++++++++++++++ .../compiler/toolchain/SQLVisitor.java | 17 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 26 ++- .../jdbc/qoq/QoQSelectStatementExecution.java | 2 +- .../runtime/jdbc/qoq/QoQStatement.java | 4 + .../ortus/boxlang/compiler/QoQParseTest.java | 60 +++++++ 7 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 7a0ee42bb..7268463b5 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -260,6 +260,7 @@ expr: | expr NOT_? IN_ ( // OPEN_PAR (select_stmt | expr ( COMMA expr)*)? CLOSE_PAR OPEN_PAR (expr ( COMMA expr)*)? CLOSE_PAR + | subquery_no_alias // | ( schema_name DOT)? table_name // | (schema_name DOT)? table_function_name OPEN_PAR (expr (COMMA expr)*)? CLOSE_PAR ) @@ -392,6 +393,10 @@ subquery: OPEN_PAR select_stmt CLOSE_PAR AS_? table_alias ; +subquery_no_alias: + OPEN_PAR select_stmt CLOSE_PAR +; + table_or_subquery: table | subquery diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java new file mode 100644 index 000000000..dec7d4031 --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java @@ -0,0 +1,149 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select.expression.operation; + +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.sql.select.SQLSelectStatement; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.operators.EqualsEquals; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Query; + +/** + * Node class representing SQL IN operation based on a sub query + */ +public class SQLInSubQueryOperation extends SQLExpression { + + private boolean not; + + private SQLExpression expression; + + // Must be an independant sub query for now. Cannot correlate to the parent query + private SQLSelectStatement subquery; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLInSubQueryOperation( SQLExpression expression, SQLSelectStatement subquery, boolean not, Position position, String sourceText ) { + super( position, sourceText ); + setExpression( expression ); + setSubQuery( subquery ); + setNot( not ); + } + + /** + * Get the expression + */ + public SQLExpression getExpression() { + return expression; + } + + /** + * Set the expression + */ + public void setExpression( SQLExpression expression ) { + replaceChildren( this.expression, expression ); + this.expression = expression; + this.expression.setParent( this ); + } + + /** + * Get the sub query + */ + public SQLSelectStatement getSubQuery() { + return subquery; + } + + /** + * Set the sub query + */ + public void setSubQuery( SQLSelectStatement subquery ) { + replaceChildren( this.subquery, subquery ); + this.subquery = subquery; + this.subquery.setParent( this ); + } + + /** + * Get the not + */ + public boolean isNot() { + return not; + } + + /** + * Set the not + */ + public void setNot( boolean not ) { + this.not = not; + } + + /** + * Runtime check if the expression evaluates to a boolean value and works for columns as well + * + * @param QoQExec Query execution state + * + * @return true if the expression evaluates to a boolean value + */ + public boolean isBoolean( QoQSelectExecution QoQExec ) { + return true; + } + + /** + * Evaluate the expression + */ + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { + Object value = expression.evaluate( QoQExec, intersection ); + Query subResult = QoQExec.getIndepententSubQuery( subquery ); + Key firstAndOnlyColName = subResult.getColumns().keySet().iterator().next(); + for ( Object v : subResult.getColumnData( firstAndOnlyColName ) ) { + if ( EqualsEquals.invoke( value, v, true ) ) { + return !not; + } + } + return not; + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + map.put( "not", not ); + map.put( "expression", expression.toMap() ); + map.put( "subquery", subquery.toMap() ); + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index b17cfed0d..bf0b0c00e 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -33,6 +33,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBinaryOperation; import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBinaryOperator; import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInSubQueryOperation; import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperation; import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperator; import ortus.boxlang.compiler.parser.SQLParser; @@ -350,7 +351,7 @@ public SQLTableVariable visitTable( TableContext ctx ) { public SQLTableSubQuery visitSubquery( SubqueryContext ctx ) { var pos = tools.getPosition( ctx ); var src = tools.getSourceText( ctx ); - SQLSelectStatement select = ( SQLSelectStatement ) visit( ctx.select_stmt() ); + SQLSelectStatement select = ( SQLSelectStatement ) new SQLVisitor( tools ).visit( ctx.select_stmt() ); return new SQLTableSubQuery( select, ctx.table_alias().getText(), tableIndex++, pos, src ); } @@ -472,9 +473,17 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j } return new SQLParam( name, index, pos, src ); } else if ( ctx.IN_() != null ) { - SQLExpression expr = visitExpr( ctx.expr( 0 ), table, joins ); - List values = ctx.expr().stream().skip( 1 ).map( e -> visitExpr( e, table, joins ) ).toList(); - return new SQLInOperation( expr, values, ctx.NOT_() != null, pos, src ); + SQLExpression expr = visitExpr( ctx.expr( 0 ), table, joins ); + if ( ctx.subquery_no_alias() != null ) { + SQLSelectStatement subquery = ( SQLSelectStatement ) new SQLVisitor( tools ).visit( ctx.subquery_no_alias().select_stmt() ); + if ( subquery.getSelect().getResultColumns().size() != 1 ) { + tools.reportError( "Subquery in IN clause must return exactly one column", pos ); + } + return new SQLInSubQueryOperation( expr, subquery, ctx.NOT_() != null, pos, src ); + } else { + List values = ctx.expr().stream().skip( 1 ).map( e -> visitExpr( e, table, joins ) ).toList(); + return new SQLInOperation( expr, values, ctx.NOT_() != null, pos, src ); + } } else if ( ctx.LIKE_() != null ) { SQLBinaryOperator op = ctx.NOT_() != null ? SQLBinaryOperator.NOTLIKE : SQLBinaryOperator.LIKE; SQLExpression escape = null; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index 6efd7cf46..0cddd2f19 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import ortus.boxlang.compiler.ast.sql.select.SQLResultColumn; import ortus.boxlang.compiler.ast.sql.select.SQLSelect; @@ -41,10 +42,12 @@ */ public class QoQSelectExecution { - public SQLSelect select; - public Map resultColumns = null; - public Map tableLookup; - public QoQSelectStatementExecution selectStatementExecution; + public SQLSelect select; + public Map resultColumns = null; + public Map tableLookup; + public QoQSelectStatementExecution selectStatementExecution; + + private Map independentSubQueries = new ConcurrentHashMap(); /** * Constructor @@ -183,4 +186,19 @@ public void calculateOrderBys() { QoQStmtExec.setAdditionalColumns( additionalColumns ); } + /** + * Indepenant sub queries are not based on the context of the outer query and can be cached here. + * + * @param subquery + * + * @return + */ + public Query getIndepententSubQuery( SQLSelectStatement subquery ) { + return independentSubQueries.computeIfAbsent( + subquery, + sq -> QoQExecutionService.executeSelectStatement( selectStatementExecution.getJDBCStatement().getContext(), sq, + selectStatementExecution.getJDBCStatement() ) + ); + } + } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java index 975f54b3d..af2da9400 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java @@ -38,7 +38,7 @@ public class QoQSelectStatementExecution { public List orderByColumns = null; public Set additionalColumns = null; public List selects = new ArrayList(); - QoQStatement JDBCStatement; + public QoQStatement JDBCStatement; /** * Constructor diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java index dd0b8079d..be71fdb59 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQStatement.java @@ -35,6 +35,10 @@ public QoQStatement( IBoxContext context, Connection connection ) { this.connection = connection; } + public IBoxContext getContext() { + return context; + } + public ResultSet executeQuery( String sql ) throws SQLException { throw new UnsupportedOperationException(); } diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 10d66f2c6..ce92a7873 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -146,4 +146,64 @@ select col as brad from ( context ); } + @Test + public void testSubquery2() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200],["Janitor",200]] ) + q = queryExecute( " + select e.*, s.name as supName, d.name as deptname + from (select * from qryEmployees) e + inner join (select * from qryEmployees) s on e.supervisor = s.name + full join (select * from qryDept) d on e.dept = d.name + where d.name in ('IT','HR') + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + } + + @Test + public void testInSubquery() { + instance.executeSource( + """ + + qryMen = queryNew( "name", "varchar", [["Luis"],["Jon"],["Brad"]] ) + qryAll = queryNew( "name", "varchar", [["Luis"],["Jon"],["Brad"],["Esme"],["Myrna"]] ) + q = queryExecute( " + select * + from qryAll + where name in ( select name from qryMen ) + + ", + [], + { dbType : "query" } + ); + println( q ) + q = queryExecute( " + select * + from qryAll + where name not in ( select name from qryMen ) + + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + } + } From 3244a3067ef8c12abef3f243f0aa1fa6787ee248 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 14 Dec 2024 22:17:49 -0600 Subject: [PATCH 008/161] BL-823 --- .../runtime/jdbc/qoq/QoQFunctionService.java | 10 ++++++++ .../ortus/boxlang/compiler/QoQParseTest.java | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index 04cc24886..f14bdf597 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -19,6 +19,7 @@ import java.util.Map; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Max; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Abs; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Acos; @@ -92,6 +93,15 @@ public static void register( Key name, java.util.function.Function, functions.put( name, QoQFunction.of( function, returnType, requiredParams ) ); } + public static void registerCustom( Key name, ortus.boxlang.runtime.types.Function function, QueryColumnType returnType, int requiredParams, + IBoxContext context ) { + functions.put( name, QoQFunction.of( + ( List arguments ) -> context.invokeFunction( function, arguments.toArray() ), + returnType, + requiredParams + ) ); + } + public static void registerAggregate( Key name, java.util.function.BiFunction, QoQSelectExecution, Object> function, QueryColumnType returnType, int requiredParams ) { diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index ce92a7873..e8821c69f 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -206,4 +206,27 @@ where name not in ( select name from qryMen ) context ); } + @Test + public void testcustomFunc() { + instance.executeSource( + """ + import ortus.boxlang.runtime.jdbc.qoq.QoQFunctionService; + import ortus.boxlang.runtime.scopes.Key; + import ortus.boxlang.runtime.types.QueryColumnType; + + // Register a custom function + QoQFunctionService.registerCustom( Key.of("reverse"), ::reverse, QueryColumnType.VARCHAR, 1, getBoxContext() ); + + q = queryExecute( " + select reverse( 'Brad' ) as rev + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + } From 9876411e3bc6e3cbfc219cc5689be2d45d1decb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:25:07 +0000 Subject: [PATCH 009/161] Bump org.apache.commons:commons-text from 1.12.0 to 1.13.0 Bumps org.apache.commons:commons-text from 1.12.0 to 1.13.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 21898728a..4f9ec979c 100644 --- a/build.gradle +++ b/build.gradle @@ -114,7 +114,7 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.17.0' // https://mvnrepository.com/artifact/org.apache.commons/commons-text // Many of these classes ( e.g. StringEscapeUtils ) are currently deprecated in commons-lang and others will be moved in the future - implementation 'org.apache.commons:commons-text:1.12.0' + implementation 'org.apache.commons:commons-text:1.13.0' // https://mvnrepository.com/artifact/org.apache.commons/commons-cli implementation "commons-cli:commons-cli:1.9.0" // https://mvnrepository.com/artifact/com.fasterxml.jackson.jr/jackson-jr-objects From 360e81915d746fae710db7f50862644623513bef Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 16 Dec 2024 22:29:43 +0100 Subject: [PATCH 010/161] some docs --- .../java/ortus/boxlang/runtime/types/meta/ClassMeta.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java b/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java index 23770629f..499eebb6c 100644 --- a/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java +++ b/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java @@ -58,10 +58,8 @@ public ClassMeta( IClassRunnable target ) { compileTimeMethodNames .stream() .map( variablesScope::get ) - .filter( entry -> entry instanceof Function ) - .forEach( entry -> { - functions.add( ( ( FunctionMeta ) ( ( Function ) entry ).getBoxMeta() ).meta ); - } ); + .filter( Function.class::isInstance ) + .forEach( entry -> functions.add( ( ( FunctionMeta ) ( ( Function ) entry ).getBoxMeta() ).meta ) ); this.meta = UnmodifiableStruct.of( Key._NAME, target.bxGetName().getName(), From 0b90d16cc8a69b310c57ba6ff64378e1c0d9decf Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 16 Dec 2024 23:38:20 +0100 Subject: [PATCH 011/161] BL-851 #resolve Java List dumps are not working with top and are off by 1 --- src/main/resources/dump/html/List.bxm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/resources/dump/html/List.bxm b/src/main/resources/dump/html/List.bxm index fa53d1fac..c965fd696 100644 --- a/src/main/resources/dump/html/List.bxm +++ b/src/main/resources/dump/html/List.bxm @@ -13,14 +13,15 @@ > #label# - - #var.getClass().getSimpleName()#: #var.size()# items + #var.getClass().getName()# + #top#/#var.size()# - + #label# - - #var.getClass().getSimpleName()#: #var.size()# items + #var.getClass().getSimpleName()# @@ -30,7 +31,7 @@ for ( i = 0; i < var.size(); i++ ) { // Top limit only if > 0 - if( !isNull( top ) && i > top ) { + if( !isNull( top ) && i >= top ) { break; } ``` From 5b37198a499afd53b1de3a7c6998e31954fae5c6 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 16 Dec 2024 23:40:09 +0100 Subject: [PATCH 012/161] BL-852 #resolve dump Lots of UI quality of life improvements: show length of strings, show full classes for some java integrations, and much more. --- src/main/java/ortus/boxlang/runtime/util/DumpUtil.java | 2 +- src/main/resources/dump/html/Array.bxm | 4 ++-- src/main/resources/dump/html/Number.bxm | 2 +- src/main/resources/dump/html/String.bxm | 2 +- src/main/resources/dump/html/Struct.bxm | 10 +++++----- src/main/resources/dump/html/ToString.bxm | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/util/DumpUtil.java b/src/main/java/ortus/boxlang/runtime/util/DumpUtil.java index aa15ca5b6..239d46b4d 100644 --- a/src/main/java/ortus/boxlang/runtime/util/DumpUtil.java +++ b/src/main/java/ortus/boxlang/runtime/util/DumpUtil.java @@ -373,7 +373,7 @@ public static void dumpHTMLToBuffer( posInCode = ExceptionUtil.getCurrentPositionInCode(); // This assumes HTML output. Needs to be dynamic as XML or plain text output wouldn't have CSS dumpContext.writeToBuffer( "", true ); - dumpContext.writeToBuffer( "" ); + dumpContext.writeToBuffer( "", true ); } // Place the variables in the scope diff --git a/src/main/resources/dump/html/Array.bxm b/src/main/resources/dump/html/Array.bxm index db02b83bb..942452c64 100644 --- a/src/main/resources/dump/html/Array.bxm +++ b/src/main/resources/dump/html/Array.bxm @@ -14,14 +14,14 @@ #label# - Array: - #top#/#var.len()# items + #top#/#var.len()# #label# - - Array: #var.len()# items + Array: #var.len()# diff --git a/src/main/resources/dump/html/Number.bxm b/src/main/resources/dump/html/Number.bxm index 92609f54b..2c3820192 100644 --- a/src/main/resources/dump/html/Number.bxm +++ b/src/main/resources/dump/html/Number.bxm @@ -3,7 +3,7 @@ #label# - - Number: + Number [#var.getClass().getSimpleName()#]: #encodeForHTML( var )# diff --git a/src/main/resources/dump/html/String.bxm b/src/main/resources/dump/html/String.bxm index 88e83c2d8..473c5cb00 100644 --- a/src/main/resources/dump/html/String.bxm +++ b/src/main/resources/dump/html/String.bxm @@ -3,7 +3,7 @@ #label# - - String: + String [#var.len()#]: #encodeForHTML( var )# diff --git a/src/main/resources/dump/html/Struct.bxm b/src/main/resources/dump/html/Struct.bxm index e8f486bb3..e0db129f7 100644 --- a/src/main/resources/dump/html/Struct.bxm +++ b/src/main/resources/dump/html/Struct.bxm @@ -19,9 +19,9 @@ #label# - #encodeForHTML( var.getName().getName().toUpperCase() )# Scope: - #top#/#var.getDumpKeys().size()# items + #top#/#var.getDumpKeys().size()# - #top#/#var.len()# items + #top#/#var.len()# @@ -30,7 +30,7 @@ #label# - #encodeForHTML( var.getName().getName().toUpperCase() )# Scope: - #top#/#var.len()# items + #top#/#var.len()# @@ -47,7 +47,7 @@ #label# - Struct: - #top#/#var.len()# items + #top#/#var.len()# @@ -55,7 +55,7 @@ #label# - Struct: - #top#/#var.len()# items + #top#/#var.len()# diff --git a/src/main/resources/dump/html/ToString.bxm b/src/main/resources/dump/html/ToString.bxm index e422a183c..4315c62f1 100644 --- a/src/main/resources/dump/html/ToString.bxm +++ b/src/main/resources/dump/html/ToString.bxm @@ -6,7 +6,7 @@ #encodeForHTML( label )# - #var.getClass().getName()# + #var.getClass().getCanonicalName()# From 5a94bed7890e9f77f2ddd01cba86fc3de56d1f29 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 16 Dec 2024 23:40:27 +0100 Subject: [PATCH 013/161] BL-852 #resolve dump Lots of UI quality of life improvements: show length of strings, show full classes for some java integrations, and much more. --- src/main/resources/dump/html/Map.bxm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/dump/html/Map.bxm b/src/main/resources/dump/html/Map.bxm index 0f468884b..be3c19780 100644 --- a/src/main/resources/dump/html/Map.bxm +++ b/src/main/resources/dump/html/Map.bxm @@ -13,7 +13,7 @@ > #label# - - #var.getClass().getSimpleName()#: + #var.getClass().getName()#: #top#/#var.size()# items @@ -21,7 +21,7 @@ #label# - - #var.getClass().getSimpleName()#: + #var.getClass().getName()#: #top#/#var.size()# items From 032d23168377a6f1327cb124e72f55a99e5fe9b2 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 00:11:16 +0100 Subject: [PATCH 014/161] BL-854 #resolve Wirebox Block: testbuildJavaClass Methods on Java objects cannot be called with named arguments when using invoke() --- .../runtime/bifs/global/system/Invoke.java | 13 ++++++- .../runtime/components/system/Invoke.java | 15 +++++++ .../interop/DynamicInteropService.java | 10 +++++ .../runtime/components/system/InvokeTest.java | 39 +++++++++++++------ 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java index ba88f0445..6f790b317 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java @@ -21,11 +21,13 @@ import ortus.boxlang.runtime.dynamic.IReferenceable; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; import ortus.boxlang.runtime.dynamic.casters.StringCaster; +import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.loader.ClassLocator; import ortus.boxlang.runtime.runnables.IClassRunnable; import ortus.boxlang.runtime.scopes.ArgumentsScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Argument; +import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; import ortus.boxlang.runtime.types.exceptions.BoxValidationException; @@ -89,7 +91,16 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { throw new BoxValidationException( "The instance parameter must be a Box Class, referencable struct or the name of a Box Class to instantiate." ); } - // Invoke the method on the Box Class instance + // ALERT! + // Special Case: If the instance is a DynamicObject and the method is "init", we need to call the constructor + if ( actualInstance instanceof DynamicObject castedDo && methodname.equals( Key.init ) ) { + // The incoming args must be an array or throw an exception + if ( ! ( args instanceof Array castedArray ) ) { + throw new BoxValidationException( "The arguments must be an array in order to execute the Java constructor." ); + } + return castedDo.invokeConstructor( context, castedArray.toArray() ); + } + return actualInstance.dereferenceAndInvoke( context, methodname, argCollection, false ); } diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Invoke.java b/src/main/java/ortus/boxlang/runtime/components/system/Invoke.java index 6b45bf47f..28f660837 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Invoke.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Invoke.java @@ -28,9 +28,11 @@ import ortus.boxlang.runtime.dynamic.IReferenceable; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; import ortus.boxlang.runtime.dynamic.casters.StringCaster; +import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.loader.ClassLocator; import ortus.boxlang.runtime.runnables.IClassRunnable; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; import ortus.boxlang.runtime.types.exceptions.BoxValidationException; @@ -120,12 +122,25 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod throw new BoxValidationException( "The instance parameter must be a Box Class or the name of a Box Class to instantiate." ); } + // ALERT! + // Special Case: If the instance is a DynamicObject and the method is "init", we need to call the constructor + if ( actualInstance instanceof DynamicObject castedDo && methodname.equals( Key.init ) ) { + // The incoming args must be an array or throw an exception + if ( ! ( args instanceof Array castedArray ) ) { + throw new BoxValidationException( "The arguments must be an array in order to execute the Java constructor." ); + } + castedDo.invokeConstructor( context, castedArray.toArray() ); + return DEFAULT_RETURN; + } + // Invoke the method on the Box Class instance result = actualInstance.dereferenceAndInvoke( context, methodname, argCollection, false ); } + if ( returnVariable != null ) { ExpressionInterpreter.setVariable( context, returnVariable, result ); } + return DEFAULT_RETURN; } diff --git a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java index c4ea734c1..0bc4381f6 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java +++ b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java @@ -1990,6 +1990,16 @@ public static Object dereferenceAndInvoke( } } + // Check if the namedArguments has the `argumentCollection` key. + // And if it's an Array, delegeate to the positional arguments + // This use-case can come from invoke( javaObject, "method", [1, 2, 3] ) + if ( namedArguments.containsKey( Key.argumentCollection ) ) { + Object argumentCollection = namedArguments.get( Key.argumentCollection ); + if ( argumentCollection instanceof Array castedArray ) { + return dereferenceAndInvoke( dynamicObject, targetClass, targetInstance, context, name, castedArray.toArray(), safe ); + } + } + throw new BoxRuntimeException( "Methods on Java objects cannot be called with named arguments" ); } diff --git a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java index d2dad20b4..a3b072fef 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java @@ -20,7 +20,8 @@ import static com.google.common.truth.Truth.assertThat; -import org.junit.jupiter.api.AfterAll; +import java.util.LinkedHashMap; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -30,6 +31,7 @@ import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; +import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.VariablesScope; @@ -46,28 +48,43 @@ public static void setUp() { instance = BoxRuntime.getInstance( true ); } - @AfterAll - public static void teardown() { - - } - @BeforeEach public void setupEach() { context = new ScriptingRequestBoxContext( instance.getRuntimeContext() ); variables = context.getScopeNearby( VariablesScope.name ); } + @DisplayName( "Call a Java Class" ) + @Test + public void testInvokeJavaClass() { + // @formatter:off + instance.executeSource( + """ + result = invoke( + createObject( "java", "java.util.LinkedHashMap"), + "init", + [ 3, 5 ] + ) + """, + context ); + // @formatter:on + DynamicObject sut = ( DynamicObject ) variables.get( result ); + assertThat( sut.unWrap() ).isInstanceOf( LinkedHashMap.class ); + } + @DisplayName( "It can invoke in current context" ) @Test public void testInvokeCurrentContext() { + // @formatter:off instance.executeSource( """ - function foo() { - return "bar"; - } - invoke method="foo" returnVariable="result"; - """, + function foo() { + return "bar"; + } + invoke method="foo" returnVariable="result"; + """, context ); + // @formatter:on assertThat( variables.get( result ) ).isEqualTo( "bar" ); } From 9dd6a35fc632c053df57369a034c7d083d8c7b6e Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 13:47:59 +0100 Subject: [PATCH 015/161] BL-856 #resolve Coercion from strings to numbers does not exist BL-855 #resolve Coercion for constructors from string to numbers are not working --- .../interop/DynamicInteropService.java | 464 ++++++++++-------- .../runtime/components/system/InvokeTest.java | 18 + .../interop/DynamicInteropServiceTest.java | 13 +- 3 files changed, 293 insertions(+), 202 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java index 0bc4381f6..d0056f599 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java +++ b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java @@ -283,14 +283,15 @@ public static T invokeConstructor( IBoxContext context, Class targetClass unWrapArguments( args ); // Discover the constructor method handle using the target class and the argument type matching - MethodHandle constructorHandle; + MethodHandle constructorHandle; + Class[] argumentClasses = argumentsToClasses( args ); try { constructorHandle = METHOD_LOOKUP.unreflectConstructor( - findMatchingConstructor( context, targetClass, argumentsToClasses( args ), args ) + findMatchingConstructor( context, targetClass, argumentClasses, args ) ); } catch ( IllegalAccessException e ) { throw new BoxRuntimeException( - "Error getting constructor for class " + targetClass.getName() + " with arguments classes " + Arrays.toString( argumentsToClasses( args ) ), + "Error getting constructor for class " + targetClass.getName() + " with arguments classes " + Arrays.toString( argumentClasses ), e ); } @@ -388,157 +389,6 @@ public static T invokeConstructor( IBoxContext context, Class targetClass return invokeConstructor( context, targetClass, EMPTY_ARGS ); } - /** - * This method is used to make sure BoxLang classes are loaded correctly with their appropriate: - * - Super class - * - Interfaces - * - Abstract methods - * - Constructor - * - Imports - * - Etc. - * - * @param context The context to use for the constructor - * @param boxClass The class to bootstrap - * @param positionalArgs The positional arguments to pass to the constructor - * @param namedArgs The named arguments to pass to the constructor - * @param noInit Whether to skip the initialization of the class or not - * - * @return The instance of the class - */ - @SuppressWarnings( "unchecked" ) - private static T bootstrapBLClass( IBoxContext context, IClassRunnable boxClass, Object[] positionalArgs, Map namedArgs, boolean noInit ) { - // This class context is really only used while boostrapping the pseudoConstructor. It will NOT be used as a parent - // context once the boxClass is initialized. Methods called on this boxClass will have access to the variables/this scope via their - // FunctionBoxContext, but their parent context will be whatever context they are called from. - IBoxContext classContext = new ClassBoxContext( context, boxClass ); - // Bootstrap the pseudoConstructor - classContext.pushTemplate( boxClass ); - - try { - // First, we load the super class if it exists - Object superClassObject = boxClass.getAnnotations().get( Key._EXTENDS ); - if ( superClassObject != null ) { - String superClassName = StringCaster.cast( superClassObject ); - if ( superClassName != null && superClassName.length() > 0 && !superClassName.toLowerCase().startsWith( "java:" ) ) { - // Recursively load the super class - IClassRunnable _super = ( IClassRunnable ) getClassLocator().load( classContext, - superClassName, - classContext.getCurrentImports() - ) - // Constructor args are NOT passed. Only the outermost class gets to use those - .invokeConstructor( classContext, new Object[] { Key.noInit } ) - .unWrapBoxLangClass(); - - // Check for final annotation and throw if we're trying to extend a final class - if ( _super.getAnnotations().get( Key._final ) != null ) { - throw new BoxRuntimeException( "Cannot extend final class: " + _super.bxGetName() ); - } - // Set in our super class - boxClass.setSuper( _super ); - } - } - - // Run the pseudo constructor - boxClass.pseudoConstructor( classContext ); - - // Now that UDFs are defined, let's enforce any interfaces - Object oInterfaces = boxClass.getAnnotations().get( Key._IMPLEMENTS ); - if ( oInterfaces != null ) { - List interfaceNames = ListUtil.asList( StringCaster.cast( oInterfaces ), "," ) - .stream() - .map( String::valueOf ) - .map( String::trim ) - // ignore anything starting with java: (case insensitive) - .filter( name -> !name.toLowerCase().startsWith( "java:" ) ) - .toList(); - - for ( String interfaceName : interfaceNames ) { - BoxInterface thisInterface = ( BoxInterface ) getClassLocator().load( classContext, interfaceName, classContext.getCurrentImports() ) - .unWrapBoxLangClass(); - boxClass.registerInterface( thisInterface ); - } - - } - - if ( !noInit ) { - if ( boxClass.getAnnotations().get( Key._ABSTRACT ) != null ) { - throw new AbstractClassException( "Cannot instantiate an abstract class: " + boxClass.bxGetName() ); - } - if ( boxClass.getSuper() != null ) { - BoxClassSupport.validateAbstractMethods( boxClass, boxClass.getSuper().getAllAbstractMethods() ); - } - // Call constructor - // look for initMethod annotation - Object initMethod = boxClass.getAnnotations().get( Key.initMethod ); - Key initKey; - if ( initMethod != null ) { - initKey = Key.of( StringCaster.cast( initMethod ) ); - } else { - initKey = Key.init; - } - if ( boxClass.dereference( context, initKey, true ) != null ) { - Object result; - if ( positionalArgs != null ) { - result = boxClass.dereferenceAndInvoke( classContext, initKey, positionalArgs, false ); - } else { - result = boxClass.dereferenceAndInvoke( classContext, initKey, namedArgs, false ); - } - // CF returns the actual result of the constructor, but I'm not sure it makes sense or if people actually ever - // return anything other than "this". - if ( result != null ) { - // This cast will fail if the init returns something like a string - return ( T ) result; - } - } else { - // implicit constructor - - if ( positionalArgs != null && positionalArgs.length == 1 && positionalArgs[ 0 ] instanceof IStruct named ) { - namedArgs = named; - } else if ( positionalArgs != null && positionalArgs.length > 0 ) { - throw new BoxRuntimeException( "Implicit constructor only accepts named args or a single Struct as a positional arg." ); - } - - if ( namedArgs != null ) { - if ( namedArgs.containsKey( Key.argumentCollection ) && namedArgs.get( Key.argumentCollection ) instanceof IStruct argCollection ) { - // Create copy of named args, merge in argCollection without overwriting, and delete arg collection key from copy of namedargs - namedArgs = new HashMap<>( namedArgs ); - for ( Map.Entry entry : argCollection.entrySet() ) { - if ( !namedArgs.containsKey( entry.getKey() ) ) { - namedArgs.put( entry.getKey(), entry.getValue() ); - } - } - namedArgs.remove( Key.argumentCollection ); - } else if ( namedArgs.containsKey( Key.argumentCollection ) && namedArgs.get( Key.argumentCollection ) instanceof Array argArray ) { - // Create copy of named args, merge in argCollection without overwriting, and delete arg collection key from copy of namedargs - namedArgs = new HashMap<>( namedArgs ); - for ( int i = 0; i < argArray.size(); i++ ) { - namedArgs.put( Key.of( i + 1 ), argArray.get( i ) ); - } - namedArgs.remove( Key.argumentCollection ); - } - // loop over args and invoke setter methods for each - for ( Map.Entry entry : namedArgs.entrySet() ) { - // not a great way to pre-create/cache these keys since they're really based on whatever crazy args the user gives us. - // If this becomes a performance issue, we can look at caching the expected keys in the boxClass in a map where the key is the - // propery - // name - // and - // the value is the key of the setter (basically the inverse of the setterlookup map) - boxClass.dereferenceAndInvoke( classContext, Key.of( "set" + entry.getKey().getName() ), new Object[] { entry.getValue() }, false ); - } - } - } - } - } finally { - // This is for any output written in the pseudoconstructor that needs to be flushed - classContext.flushBuffer( false ); - classContext.popTemplate(); - } - - // We have a fully initialized class, so we can return it - return ( T ) boxClass; - } - /** * Invoke can be used to invoke public methods on instances, or static methods on classes/interfaces. * @@ -609,7 +459,13 @@ public static Object invoke( IBoxContext context, Class targetClass, Object t * * @return The result of the method invocation */ - public static Object invoke( IBoxContext context, DynamicObject dynamicObject, Class targetClass, Object targetInstance, String methodName, Boolean safe, + public static Object invoke( + IBoxContext context, + DynamicObject dynamicObject, + Class targetClass, + Object targetInstance, + String methodName, + Boolean safe, Object... arguments ) { // Verify method name if ( methodName == null || methodName.isEmpty() ) { @@ -1259,13 +1115,24 @@ public static MethodRecord discoverMethodHandle( * Get a HashSet of constructors of the given class * * @param targetClass The class to get the constructors for + * @param callable Whether to get only callable constructors (anything but private). If null or false all constructors are returned * * @return A unique set of callable constructors */ - public static Set> getConstructors( Class targetClass ) { + public static Set> getConstructors( Class targetClass, Boolean callable ) { Set> allConstructors = new HashSet<>(); + + // Collect all the constructors from the class and it's super classes allConstructors.addAll( new HashSet<>( List.of( targetClass.getConstructors() ) ) ); allConstructors.addAll( new HashSet<>( List.of( targetClass.getDeclaredConstructors() ) ) ); + + // If callable, filter out the private constructors + if ( Boolean.TRUE.equals( callable ) ) { + allConstructors = allConstructors.stream() + .filter( method -> Modifier.isPublic( method.getModifiers() ) ) + .collect( Collectors.toSet() ); + } + return allConstructors; } @@ -1273,11 +1140,12 @@ public static Set> getConstructors( Class targetClass ) { * Get a stream of constructors of the given class * * @param targetClass The class to get the constructors for + * @param callable Whether to get only callable constructors (anything but private). If null or false all constructors are returned * * @return A stream of unique callable constructors */ - public static Stream> getConstructorsAsStream( Class targetClass ) { - return getConstructors( targetClass ).stream(); + public static Stream> getConstructorsAsStream( Class targetClass, Boolean callable ) { + return getConstructors( targetClass, callable ).stream(); } /** @@ -1291,18 +1159,53 @@ public static Stream> getConstructorsAsStream( Class targetCla * @return The constructor if it exists */ public static Constructor findMatchingConstructor( IBoxContext context, Class targetClass, Class[] argumentsAsClasses, Object... arguments ) { - return getConstructorsAsStream( targetClass ) - // has to have the same number of arguments - .filter( constructor -> constructorHasMatchingParameterTypes( context, constructor, argumentsAsClasses, arguments ) ) - .findFirst() - .orElseThrow( () -> new NoConstructorException( - String.format( - "No such constructor found in the class [%s] using [%d] arguments of types [%s]", - targetClass.getName(), - argumentsAsClasses.length, - Arrays.toString( argumentsAsClasses ) - ) - ) ); + List> targetConstructors = getConstructorsAsStream( targetClass, true ) + // Try matches using our algorithms + .filter( constructor -> { + // No Var Args + if ( !constructor.isVarArgs() && constructor.getParameterCount() == argumentsAsClasses.length ) { + return true; + } + + // Check for var args as the last parameter + if ( constructor.isVarArgs() ) { + return argumentsAsClasses.length >= constructor.getParameterCount() - 1; + } + + // No match + return false; + } ) + .toList(); + + // Only process if we have constructors + if ( !targetConstructors.isEmpty() ) { + // 1. Exact Match : Matches the incoming argument class types to the constructor signature + Constructor foundConstructor = targetConstructors + .stream() + // Has to have the SAME arg types + .filter( constructor -> constructorHasMatchingParameterTypes( context, constructor, true, argumentsAsClasses, arguments ) ) + .findFirst() + // If no exact match, try loose coercion + .or( () -> targetConstructors + .stream() + .filter( constructor -> constructorHasMatchingParameterTypes( context, constructor, false, argumentsAsClasses, arguments ) ) + .findFirst() + ) + .orElse( null ); + + if ( foundConstructor != null ) { + return foundConstructor; + } + } + + throw new NoConstructorException( + String.format( + "No such constructor found in the class [%s] using [%d] arguments of types [%s]", + targetClass.getName(), + argumentsAsClasses.length, + Arrays.toString( argumentsAsClasses ) + ) + ); } /** @@ -1513,24 +1416,18 @@ public static Method findMatchingMethod( } // 1: Exact Match first - Method foundMethod = targetMethods + return targetMethods .stream() // has to have the same argument types .filter( method -> hasMatchingParameterTypes( context, method, true, argumentsAsClasses, arguments ) ) .findFirst() + // 2: If no exact match, try loose coercion + .or( () -> targetMethods + .stream() + .filter( method -> hasMatchingParameterTypes( context, method, false, argumentsAsClasses, arguments ) ) + .findFirst() + ) .orElse( null ); - - // 2. Loose coercion matching - if ( foundMethod == null ) { - foundMethod = targetMethods - .stream() - // Loose coercion matching next - .filter( method -> hasMatchingParameterTypes( context, method, false, argumentsAsClasses, arguments ) ) - .findFirst() - .orElse( null ); - } - - return foundMethod; } /** @@ -2071,12 +1968,6 @@ public static Object assign( IBoxContext context, Class targetClass, Object t return value; } - /** - * -------------------------------------------------------------------------- - * Private Methods - * -------------------------------------------------------------------------- - */ - /** * Verifies if the target calss is an interface or not * @@ -2088,6 +1979,12 @@ public static boolean isInterface( Class targetClass ) { return targetClass.isInterface() || BoxInterface.class.isAssignableFrom( targetClass ); } + /** + * -------------------------------------------------------------------------- + * Private Methods + * -------------------------------------------------------------------------- + */ + /** * Verifies if the method has the same parameter types as the incoming ones using our matching algorithms. * - Exact Match : Matches the incoming argument class types to the method signature @@ -2253,6 +2150,15 @@ private static Optional coerceAttempt( IBoxContext context, Class expected ); } + // Expected: Numeric and Actual: String + // If the expected type is a number and the actual is a string, we can TRY to coerce it + if ( Number.class.isAssignableFrom( expected ) && actualClass.equals( "string" ) ) { + // logger.debug( "Coerce attempt: Castable to Number from String " + actualClass ); + return Optional.of( + GenericCaster.cast( context, value, expectedClass, false ) + ); + } + // EXPECTED: FunctionInterfaces and/or SAMs Class functionalInterface = InterfaceProxyService.getFunctionalInterface( expected ); // If the target is a functional interface and the actual value is a Funcion or Runnable, coerce it @@ -2280,26 +2186,40 @@ private static Optional coerceAttempt( IBoxContext context, Class expected * * @param context The context to use for the method invocation * @param constructor The constructor to check + * @param exact Exact matching or loose matching with coercion * @param argumentsAsClasses The arguments to check * @param arguments The arguments to pass to the constructor * - * @return True if the constructor has the same parameter types, false otherwise + * @return True if the constructor has the same or coerced parameter types, false otherwise */ - private static boolean constructorHasMatchingParameterTypes( IBoxContext context, Constructor constructor, Class[] argumentsAsClasses, + private static boolean constructorHasMatchingParameterTypes( + IBoxContext context, + Constructor constructor, + Boolean exact, + Class[] argumentsAsClasses, Object... arguments ) { + + // Get param types to test from the constructor Class[] constructorParams = constructor.getParameterTypes(); - // If we have a different number of parameters, then we don't have a match - if ( constructorParams.length != argumentsAsClasses.length ) { - return false; + // Verify assignability including primitive autoboxing + if ( ClassUtils.isAssignable( argumentsAsClasses, constructorParams ) ) { + return true; } - // Verify assignability including primitive autoboxing - return ClassUtils.isAssignable( argumentsAsClasses, constructorParams ); + // Let's do coercive matching if we get here. ONLY if we are not doing an exact match + // iterate over the constructor params and check if the arguments can be coerced to the constructor params + // Every argument must be coercable or it fails + if ( !exact ) { + return coerceArguments( context, constructorParams, argumentsAsClasses, arguments, constructor.isVarArgs() ); + } + + return false; } /** * Lazy load this to avoid static intitlizer deadlocks on startup + * Most of this is done lazy to avoid static init deadlocks */ private static FunctionService getFunctionService() { if ( functionService == null ) { @@ -2314,6 +2234,7 @@ private static FunctionService getFunctionService() { /** * Lazy load ClassLocator as well + * Most of this is done lazy to avoid static init deadlocks */ private static ClassLocator getClassLocator() { if ( classLocator == null ) { @@ -2326,4 +2247,155 @@ private static ClassLocator getClassLocator() { return classLocator; } + /** + * This method is used to make sure BoxLang classes are loaded correctly with their appropriate: + * - Super class + * - Interfaces + * - Abstract methods + * - Constructor + * - Imports + * - Etc. + * + * @param context The context to use for the constructor + * @param boxClass The class to bootstrap + * @param positionalArgs The positional arguments to pass to the constructor + * @param namedArgs The named arguments to pass to the constructor + * @param noInit Whether to skip the initialization of the class or not + * + * @return The instance of the class + */ + @SuppressWarnings( "unchecked" ) + private static T bootstrapBLClass( IBoxContext context, IClassRunnable boxClass, Object[] positionalArgs, Map namedArgs, boolean noInit ) { + // This class context is really only used while boostrapping the pseudoConstructor. It will NOT be used as a parent + // context once the boxClass is initialized. Methods called on this boxClass will have access to the variables/this scope via their + // FunctionBoxContext, but their parent context will be whatever context they are called from. + IBoxContext classContext = new ClassBoxContext( context, boxClass ); + // Bootstrap the pseudoConstructor + classContext.pushTemplate( boxClass ); + + try { + // First, we load the super class if it exists + Object superClassObject = boxClass.getAnnotations().get( Key._EXTENDS ); + if ( superClassObject != null ) { + String superClassName = StringCaster.cast( superClassObject ); + if ( superClassName != null && superClassName.length() > 0 && !superClassName.toLowerCase().startsWith( "java:" ) ) { + // Recursively load the super class + IClassRunnable _super = ( IClassRunnable ) getClassLocator().load( classContext, + superClassName, + classContext.getCurrentImports() + ) + // Constructor args are NOT passed. Only the outermost class gets to use those + .invokeConstructor( classContext, new Object[] { Key.noInit } ) + .unWrapBoxLangClass(); + + // Check for final annotation and throw if we're trying to extend a final class + if ( _super.getAnnotations().get( Key._final ) != null ) { + throw new BoxRuntimeException( "Cannot extend final class: " + _super.bxGetName() ); + } + // Set in our super class + boxClass.setSuper( _super ); + } + } + + // Run the pseudo constructor + boxClass.pseudoConstructor( classContext ); + + // Now that UDFs are defined, let's enforce any interfaces + Object oInterfaces = boxClass.getAnnotations().get( Key._IMPLEMENTS ); + if ( oInterfaces != null ) { + List interfaceNames = ListUtil.asList( StringCaster.cast( oInterfaces ), "," ) + .stream() + .map( String::valueOf ) + .map( String::trim ) + // ignore anything starting with java: (case insensitive) + .filter( name -> !name.toLowerCase().startsWith( "java:" ) ) + .toList(); + + for ( String interfaceName : interfaceNames ) { + BoxInterface thisInterface = ( BoxInterface ) getClassLocator().load( classContext, interfaceName, classContext.getCurrentImports() ) + .unWrapBoxLangClass(); + boxClass.registerInterface( thisInterface ); + } + + } + + if ( !noInit ) { + if ( boxClass.getAnnotations().get( Key._ABSTRACT ) != null ) { + throw new AbstractClassException( "Cannot instantiate an abstract class: " + boxClass.bxGetName() ); + } + if ( boxClass.getSuper() != null ) { + BoxClassSupport.validateAbstractMethods( boxClass, boxClass.getSuper().getAllAbstractMethods() ); + } + // Call constructor + // look for initMethod annotation + Object initMethod = boxClass.getAnnotations().get( Key.initMethod ); + Key initKey; + if ( initMethod != null ) { + initKey = Key.of( StringCaster.cast( initMethod ) ); + } else { + initKey = Key.init; + } + if ( boxClass.dereference( context, initKey, true ) != null ) { + Object result; + if ( positionalArgs != null ) { + result = boxClass.dereferenceAndInvoke( classContext, initKey, positionalArgs, false ); + } else { + result = boxClass.dereferenceAndInvoke( classContext, initKey, namedArgs, false ); + } + // CF returns the actual result of the constructor, but I'm not sure it makes sense or if people actually ever + // return anything other than "this". + if ( result != null ) { + // This cast will fail if the init returns something like a string + return ( T ) result; + } + } else { + // implicit constructor + + if ( positionalArgs != null && positionalArgs.length == 1 && positionalArgs[ 0 ] instanceof IStruct named ) { + namedArgs = named; + } else if ( positionalArgs != null && positionalArgs.length > 0 ) { + throw new BoxRuntimeException( "Implicit constructor only accepts named args or a single Struct as a positional arg." ); + } + + if ( namedArgs != null ) { + if ( namedArgs.containsKey( Key.argumentCollection ) && namedArgs.get( Key.argumentCollection ) instanceof IStruct argCollection ) { + // Create copy of named args, merge in argCollection without overwriting, and delete arg collection key from copy of namedargs + namedArgs = new HashMap<>( namedArgs ); + for ( Map.Entry entry : argCollection.entrySet() ) { + if ( !namedArgs.containsKey( entry.getKey() ) ) { + namedArgs.put( entry.getKey(), entry.getValue() ); + } + } + namedArgs.remove( Key.argumentCollection ); + } else if ( namedArgs.containsKey( Key.argumentCollection ) && namedArgs.get( Key.argumentCollection ) instanceof Array argArray ) { + // Create copy of named args, merge in argCollection without overwriting, and delete arg collection key from copy of namedargs + namedArgs = new HashMap<>( namedArgs ); + for ( int i = 0; i < argArray.size(); i++ ) { + namedArgs.put( Key.of( i + 1 ), argArray.get( i ) ); + } + namedArgs.remove( Key.argumentCollection ); + } + // loop over args and invoke setter methods for each + for ( Map.Entry entry : namedArgs.entrySet() ) { + // not a great way to pre-create/cache these keys since they're really based on whatever crazy args the user gives us. + // If this becomes a performance issue, we can look at caching the expected keys in the boxClass in a map where the key is the + // propery + // name + // and + // the value is the key of the setter (basically the inverse of the setterlookup map) + boxClass.dereferenceAndInvoke( classContext, Key.of( "set" + entry.getKey().getName() ), new Object[] { entry.getValue() }, false ); + } + } + } + } + } finally { + // This is for any output written in the pseudoconstructor that needs to be flushed + classContext.flushBuffer( false ); + classContext.popTemplate(); + } + + // We have a fully initialized class, so we can return it + return ( T ) boxClass; + } + } diff --git a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java index a3b072fef..767fc23a5 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java @@ -72,6 +72,24 @@ public void testInvokeJavaClass() { assertThat( sut.unWrap() ).isInstanceOf( LinkedHashMap.class ); } + @DisplayName( "Call a Java Class with constructor coercion" ) + @Test + public void testInvokeJavaClassWithCoercion() { + // @formatter:off + instance.executeSource( + """ + result = invoke( + createObject( "java", "java.util.LinkedHashMap"), + "init", + [ "3", 5 ] + ) + """, + context ); + // @formatter:on + DynamicObject sut = ( DynamicObject ) variables.get( result ); + assertThat( sut.unWrap() ).isInstanceOf( LinkedHashMap.class ); + } + @DisplayName( "It can invoke in current context" ) @Test public void testInvokeCurrentContext() { diff --git a/src/test/java/ortus/boxlang/runtime/interop/DynamicInteropServiceTest.java b/src/test/java/ortus/boxlang/runtime/interop/DynamicInteropServiceTest.java index d76a18179..bcacdc60d 100644 --- a/src/test/java/ortus/boxlang/runtime/interop/DynamicInteropServiceTest.java +++ b/src/test/java/ortus/boxlang/runtime/interop/DynamicInteropServiceTest.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -641,18 +642,18 @@ void testInvokeInterfaceMethodImplementedByPrivateClassInBoxLang() { } @Test - @DisplayName( "It can get all the constructors on a class" ) + @DisplayName( "It can get all the callable constructors on a class" ) void testItCanGetAllConstructors() { - Set> constructors = DynamicInteropService.getConstructors( String.class ); + Set> constructors = DynamicInteropService.getConstructors( String.class, true ); assertThat( constructors ).isNotEmpty(); - assertThat( constructors.size() ).isAtLeast( 18 ); + assertThat( constructors.size() ).isAtLeast( 15 ); } @Test - @DisplayName( "It can get all the constructors as a stream" ) + @DisplayName( "It can get all the callable constructors as a stream" ) void testItCanGetAllConstructorsAsStream() { - Set> constructors = DynamicInteropService.getConstructors( String.class ); - assertThat( constructors.stream() ).isNotEmpty(); + Stream> constructors = DynamicInteropService.getConstructorsAsStream( String.class, true ); + assertThat( constructors ).isNotNull(); } @Test From 2a67f744e18a2f3ab984f790a14509329f064fb7 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 14:26:29 +0100 Subject: [PATCH 016/161] BL-857 #resolve isInstanceOf bif and keyword are not working on createObject("java") proxies. It gives a negative result --- .../boxlang/runtime/operators/InstanceOf.java | 6 ++- .../runtime/operators/InstanceOfTest.java | 50 ++++++++++++++++--- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/operators/InstanceOf.java b/src/main/java/ortus/boxlang/runtime/operators/InstanceOf.java index df84e002c..c01311a79 100644 --- a/src/main/java/ortus/boxlang/runtime/operators/InstanceOf.java +++ b/src/main/java/ortus/boxlang/runtime/operators/InstanceOf.java @@ -51,8 +51,8 @@ public static Boolean invoke( IBoxContext context, Object left, Object right ) { if ( left == null ) { return false; } - IClassRunnable boxClass = null; + IClassRunnable boxClass = null; String type = StringCaster.cast( right ); left = DynamicObject.unWrap( left ); @@ -67,6 +67,10 @@ public static Boolean invoke( IBoxContext context, Object left, Object right ) { } // Perform exact Java type check + // Check if it's a class or instance of a class + if ( left instanceof Class castedLeft && looseClassCheck( castedLeft.getName(), type ) ) { + return true; + } if ( looseClassCheck( left.getClass().getName(), type ) ) { return true; } diff --git a/src/test/java/ortus/boxlang/runtime/operators/InstanceOfTest.java b/src/test/java/ortus/boxlang/runtime/operators/InstanceOfTest.java index 5cbbe8661..86fe6376b 100644 --- a/src/test/java/ortus/boxlang/runtime/operators/InstanceOfTest.java +++ b/src/test/java/ortus/boxlang/runtime/operators/InstanceOfTest.java @@ -25,21 +25,32 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; +import ortus.boxlang.runtime.scopes.IScope; +import ortus.boxlang.runtime.scopes.VariablesScope; public class InstanceOfTest { - IBoxContext runtimeContext = BoxRuntime.getInstance( true ).getRuntimeContext(); + BoxRuntime instance = BoxRuntime.getInstance( true ); + IBoxContext runtimeContext = instance.getRuntimeContext(); + IBoxContext context; + IScope variables; + + @BeforeEach + void setUp() { + context = new ScriptingRequestBoxContext( runtimeContext ); + variables = context.getScopeNearby( VariablesScope.name ); + } @DisplayName( "It can check java type" ) @Test void testItCanCheckType() { - IBoxContext context = new ScriptingRequestBoxContext( runtimeContext ); assertThat( InstanceOf.invoke( context, "Brad", "java.lang.String" ) ).isTrue(); // Lucee-only behavior assertThat( InstanceOf.invoke( context, "Brad", "String" ) ).isTrue(); @@ -53,15 +64,13 @@ void testItCanCheckType() { @DisplayName( "It can check java interface" ) @Test void testItCanCheckInterface() { - IBoxContext context = new ScriptingRequestBoxContext( runtimeContext ); assertThat( InstanceOf.invoke( context, new HashMap(), "java.util.Map" ) ).isTrue(); } @DisplayName( "It can check java superinterface" ) @Test void testItCanCheckSuperInterface() { - IBoxContext context = new ScriptingRequestBoxContext( runtimeContext ); - List target = new ArrayList(); + List target = new ArrayList(); assertThat( InstanceOf.invoke( context, target, "java.util.ArrayList" ) ).isTrue(); assertThat( InstanceOf.invoke( context, target, "java.util.List" ) ).isTrue(); assertThat( InstanceOf.invoke( context, target, "java.util.Collection" ) ).isTrue(); @@ -70,12 +79,39 @@ void testItCanCheckSuperInterface() { @DisplayName( "It can check java supertype" ) @Test void testItCanCheckSupertype() { - IBoxContext context = new ScriptingRequestBoxContext( runtimeContext ); - Map target = new ConcurrentHashMap(); + Map target = new ConcurrentHashMap(); assertThat( InstanceOf.invoke( context, target, "java.util.concurrent.ConcurrentHashMap" ) ).isTrue(); assertThat( InstanceOf.invoke( context, target, "java.util.HashMap" ) ).isFalse(); assertThat( InstanceOf.invoke( context, target, "java.util.AbstractMap" ) ).isTrue(); assertThat( InstanceOf.invoke( context, target, "java.lang.Object" ) ).isTrue(); } + @DisplayName( "Can check java classes" ) + @Test + void testCanCheckJavaClasses() { + // @formatter:off + instance.executeSource( + """ + target = createObject( "java", "java.util.LinkedHashMap" ) + result = isInstanceOf( target, "java.util.LinkedHashMap" ) + """, + context ); + // @formatter:on + assertThat( variables.get( "result" ) ).isEqualTo( true ); + } + + @DisplayName( "Can check java class instances" ) + @Test + void testCanCheckJavaClassInstances() { + // @formatter:off + instance.executeSource( + """ + target = createObject( "java", "java.util.LinkedHashMap" ).init() + result = isInstanceOf( target, "java.util.LinkedHashMap" ) + """, + context ); + // @formatter:on + assertThat( variables.get( "result" ) ).isEqualTo( true ); + } + } From 3b921d005a0f3ebb98752a88055d5bbac4095c22 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 18:06:13 +0100 Subject: [PATCH 017/161] BL-858 #resolve var scoping issues for bleeding scopes on class dump template --- src/main/resources/dump/html/Class.bxm | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/resources/dump/html/Class.bxm b/src/main/resources/dump/html/Class.bxm index 48e6dc240..14c027da9 100644 --- a/src/main/resources/dump/html/Class.bxm +++ b/src/main/resources/dump/html/Class.bxm @@ -20,7 +20,7 @@ > - - for( field in fields ) { + for( variables.field in fields ) { ``` - #encodeForHTML( field.getName() )# + #encodeForHTML( variables.field.getName() )# - 
#encodeForHTML( field.toString() )#
+ 
#encodeForHTML( variables.field.toString() )#
- #encodeForHTML( field.get( var ) )# + #encodeForHTML( variables.field.get( var ) )# --- Not Available --- @@ -131,12 +131,12 @@ - for( annotation in annotations ) { + for( variables.annotation in annotations ) { ``` - #annotation.annotationType().toString()# + #variables.annotation.annotationType().toString()# - 
#encodeForHTML( annotation.toString() )#
+ 
#encodeForHTML( variables.annotation.toString() )#
``` @@ -172,11 +172,11 @@ - for( constructor in constructors ) { + for( variables.constructor in constructors ) { ``` - 
#encodeForHTML( constructor.toString() )#
+ 
#encodeForHTML( variables.constructor.toString() )#
``` @@ -205,8 +205,8 @@ import java.util.Arrays; import java.util.Comparator; - methods = Arrays.stream( var.getClass().getDeclaredMethods() ) - .sorted( Comparator.comparing( m -> m.getName() ) ) + variables.methods = Arrays.stream( var.getClass().getDeclaredMethods() ) + .sorted( Comparator.comparing( m -> arguments.m.getName() ) ) .toList(); @@ -215,14 +215,14 @@ - for( method in methods ) { + for( variables.method in methods ) { ``` ``` From a547b50f76a14113786dc3a15c2dba3368e0b7b6 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 18:46:38 +0100 Subject: [PATCH 018/161] BL-859 #resolve Calling getMetadata() on an instance of dynamic object that has not yet been inited, shows the metadata of DynamicObject instead of the proxy class --- coldbox.sh | 9 +++++++++ .../boxlang/runtime/bifs/global/type/GetMetaData.java | 7 +++++++ 2 files changed, 16 insertions(+) create mode 100755 coldbox.sh diff --git a/coldbox.sh b/coldbox.sh new file mode 100755 index 000000000..c7d34378f --- /dev/null +++ b/coldbox.sh @@ -0,0 +1,9 @@ +#!/bin/sh +cd ~/Sites/projects/boxlang +gradle build -x test -x javadoc + +cd ~/Sites/projects/boxlang-web-support +gradle build -x test -x javadoc + +cd ~/Sites/projects/boxlang-servlet +gradle buildRuntime diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java index 25989c60d..45b583b08 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java @@ -17,6 +17,7 @@ import ortus.boxlang.runtime.bifs.BIF; import ortus.boxlang.runtime.bifs.BoxBIF; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.scopes.ArgumentsScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Argument; @@ -47,6 +48,12 @@ public GetMetaData() { public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { Object value = arguments.get( Key.value ); + // Do we have a DynamicObject? + // If so, we need to invoke the constructor to get the actual object + if ( value instanceof DynamicObject doObject ) { + value = doObject.invokeConstructor( context ).unWrap(); + } + // Functions have a legacy metadata view that matches CF engines if ( value instanceof IType bxObject ) { return bxObject.getBoxMeta().getMeta(); From e271fc2c4987ab02f418c83139de05d5e0f6fe90 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Tue, 17 Dec 2024 18:54:55 +0100 Subject: [PATCH 019/161] correct dynamic object wrapping --- .../ortus/boxlang/runtime/bifs/global/type/GetMetaData.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java index 45b583b08..67c8c44eb 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java @@ -51,7 +51,11 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { // Do we have a DynamicObject? // If so, we need to invoke the constructor to get the actual object if ( value instanceof DynamicObject doObject ) { - value = doObject.invokeConstructor( context ).unWrap(); + if ( doObject.hasInstance() ) { + value = doObject.unWrap(); + } else { + value = doObject.invokeConstructor( context ).unWrap(); + } } // Functions have a legacy metadata view that matches CF engines From 9ff6591f2adc8247f0eaece97e41b5941d016eaf Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Tue, 17 Dec 2024 14:10:18 -0600 Subject: [PATCH 020/161] BL-860 --- .../runtime/bifs/global/type/GetMetaData.java | 17 ++--- .../boxlang/runtime/dynamic/Referencer.java | 2 +- .../interop/DynamicInteropService.java | 66 ++++++++++++++++++- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java index 67c8c44eb..91109e2c5 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/type/GetMetaData.java @@ -48,22 +48,19 @@ public GetMetaData() { public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { Object value = arguments.get( Key.value ); - // Do we have a DynamicObject? - // If so, we need to invoke the constructor to get the actual object - if ( value instanceof DynamicObject doObject ) { - if ( doObject.hasInstance() ) { - value = doObject.unWrap(); - } else { - value = doObject.invokeConstructor( context ).unWrap(); - } - } + // DynamicObject instances need to be unwrapped to get the metadata + value = DynamicObject.unWrap( value ); // Functions have a legacy metadata view that matches CF engines if ( value instanceof IType bxObject ) { return bxObject.getBoxMeta().getMeta(); } - // All other types return the class + // All other types return the class of the value to match CF engines + if ( value instanceof Class ) { + return value; + } + return value.getClass(); } diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/Referencer.java b/src/main/java/ortus/boxlang/runtime/dynamic/Referencer.java index ba4684e40..4619168e5 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/Referencer.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/Referencer.java @@ -58,7 +58,7 @@ public static Object get( IBoxContext context, Object object, Key key, Boolean s if ( object instanceof DynamicObject dob ) { return dob.dereference( context, key, safe ); } else if ( object instanceof Class clazz ) { - return DynamicInteropService.dereference( context, clazz, null, key, safe ); + return DynamicInteropService.dereference( context, clazz, object, key, safe ); } else { return DynamicInteropService.dereference( context, object.getClass(), object, key, safe ); } diff --git a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java index d0056f599..42ea60492 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java +++ b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java @@ -813,6 +813,18 @@ public static Boolean hasFieldNoCase( Class targetClass, String fieldName ) { return getFieldNamesNoCase( targetClass ).contains( fieldName.toUpperCase() ); } + /** + * Verifies if the class has a public or public static field with the given name and no case-sensitivity (upper case) + * + * @param targetClass The class to check + * @param fieldName The name of the field to check + * + * @return True if the field exists, false otherwise + */ + public static Boolean hasPublicFieldNoCase( Class targetClass, String fieldName ) { + return getPublicFieldNamesNoCase( targetClass ).contains( fieldName.toUpperCase() ); + } + /** * Get an array of fields of all the public fields for the given class * @@ -827,6 +839,17 @@ public static Field[] getFields( Class targetClass ) { return allFields.toArray( new Field[ 0 ] ); } + /** + * Get an array of fields of all the public fields for the given class + * + * @param targetClass The class to get the fields for + * + * @return The fields in the class + */ + public static Field[] getPublicFields( Class targetClass ) { + return targetClass.getFields(); + } + /** * Get a stream of fields of all the public fields for the given class * @@ -838,6 +861,17 @@ public static Stream getFieldsAsStream( Class targetClass ) { return Stream.of( getFields( targetClass ) ); } + /** + * Get a stream of fields of all the public fields for the given class + * + * @param targetClass The class to get the fields for + * + * @return The stream of fields in the class + */ + public static Stream getPublicFieldsAsStream( Class targetClass ) { + return Stream.of( getPublicFields( targetClass ) ); + } + /** * Get a list of field names for the given class with case-sensitivity * @@ -864,7 +898,20 @@ public static List getFieldNamesNoCase( Class targetClass ) { .map( Field::getName ) .map( String::toUpperCase ) .toList(); + } + /** + * Get a list of field names for the given class with no case-sensitivity (upper case) + * + * @param targetClass The class to get the fields for + * + * @return A list of field names + */ + public static List getPublicFieldNamesNoCase( Class targetClass ) { + return getPublicFieldsAsStream( targetClass ) + .map( Field::getName ) + .map( String::toUpperCase ) + .toList(); } /** @@ -1670,7 +1717,7 @@ public static Object dereference( IBoxContext context, Class targetClass, Obj } return s.substring( index - 1, index ); // Special logic for native arrays. Possibly move to helper - } else if ( hasFieldNoCase( targetClass, name.getName() ) ) { + } else if ( hasAccessibleField( targetInstance, targetClass, name.getName() ) ) { // If we have the field, return its value, even if it's null return getField( targetClass, targetInstance, name.getName() ).orElse( null ); } else if ( hasClassNoCase( targetClass, name.getName() ) ) { @@ -1718,6 +1765,23 @@ public static Object dereference( IBoxContext context, Class targetClass, Obj } } + private static boolean hasAccessibleField( Object targetInstance, Class targetClass, String fieldName ) { + // Get all fields on the class + Optional possibleMatch = getFieldsAsStream( targetClass ).filter( field -> field.getName().equalsIgnoreCase( fieldName ) ).findFirst(); + // If the field doesn't exist, early exit + if ( !possibleMatch.isPresent() ) { + return false; + } + Field field = possibleMatch.get(); + // As sort-of dumb work around for a BoxClass's ability to reference super.protectedField when extending java, we'll assume it's accessible if accessing on a BoxClass and the field is from this class + // This doesn't really test the true calling context, but we'd need to proxy to the actual box class for that to work. + if ( targetInstance instanceof IClassRunnable && !field.getDeclaringClass().equals( targetInstance.getClass() ) ) { + return true; + } + // For all other cases, we'll just check if the field is public + return Modifier.isPublic( field.getModifiers() ); + } + /** * Dereference this object by a key and invoke the result as an invokable (UDF, java method) * From 83fb1877a958e8b56827a844a67b29ae2b992351 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Tue, 17 Dec 2024 16:46:28 -0600 Subject: [PATCH 021/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 6 +- src/main/antlr/SQLLexer.g4 | 1 + .../ast/sql/select/expression/SQLColumn.java | 11 ++ .../select/expression/SQLCountFunction.java | 41 ++++- .../sql/select/expression/SQLExpression.java | 7 + .../sql/select/expression/SQLFunction.java | 28 +++- .../ast/sql/select/expression/SQLParam.java | 11 ++ .../sql/select/expression/SQLParenthesis.java | 8 + .../select/expression/SQLStarExpression.java | 8 + .../expression/literal/SQLBooleanLiteral.java | 8 + .../expression/literal/SQLNullLiteral.java | 8 + .../expression/literal/SQLNumberLiteral.java | 8 + .../expression/literal/SQLStringLiteral.java | 8 + .../operation/SQLBetweenOperation.java | 11 ++ .../operation/SQLBinaryOperation.java | 140 +++++++++++++++++- .../expression/operation/SQLInOperation.java | 10 ++ .../operation/SQLInSubQueryOperation.java | 11 ++ .../operation/SQLUnaryOperation.java | 11 ++ .../boxlang/compiler/parser/SQLParser.java | 3 +- .../compiler/toolchain/SQLVisitor.java | 67 +++++++-- .../runtime/jdbc/qoq/IQoQFunctionDef.java | 5 +- .../jdbc/qoq/QoQAggregateFunctionDef.java | 2 +- .../runtime/jdbc/qoq/QoQExecutionService.java | 72 +++++++-- .../runtime/jdbc/qoq/QoQFunctionService.java | 60 +++++--- .../jdbc/qoq/QoQScalarFunctionDef.java | 4 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 13 ++ .../jdbc/qoq/functions/aggregate/Avg.java | 56 +++++++ .../jdbc/qoq/functions/aggregate/Max.java | 22 ++- .../jdbc/qoq/functions/aggregate/Min.java | 62 ++++++++ .../jdbc/qoq/functions/aggregate/Sum.java | 56 +++++++ .../jdbc/qoq/functions/scalar/Abs.java | 6 +- .../jdbc/qoq/functions/scalar/Acos.java | 6 +- .../jdbc/qoq/functions/scalar/Asin.java | 6 +- .../jdbc/qoq/functions/scalar/Atan.java | 6 +- .../jdbc/qoq/functions/scalar/Cast.java | 60 ++++++++ .../jdbc/qoq/functions/scalar/Ceiling.java | 6 +- .../jdbc/qoq/functions/scalar/Coalesce.java | 6 +- .../jdbc/qoq/functions/scalar/Concat.java | 6 +- .../jdbc/qoq/functions/scalar/Convert.java | 31 ++++ .../jdbc/qoq/functions/scalar/Cos.java | 6 +- .../jdbc/qoq/functions/scalar/Exp.java | 6 +- .../jdbc/qoq/functions/scalar/Floor.java | 6 +- .../jdbc/qoq/functions/scalar/IsNull.java | 6 +- .../jdbc/qoq/functions/scalar/Length.java | 6 +- .../jdbc/qoq/functions/scalar/Lower.java | 6 +- .../jdbc/qoq/functions/scalar/Ltrim.java | 6 +- .../jdbc/qoq/functions/scalar/Mod.java | 6 +- .../jdbc/qoq/functions/scalar/Power.java | 6 +- .../jdbc/qoq/functions/scalar/Rtrim.java | 6 +- .../jdbc/qoq/functions/scalar/Sin.java | 6 +- .../jdbc/qoq/functions/scalar/Sqrt.java | 6 +- .../jdbc/qoq/functions/scalar/Tan.java | 6 +- .../jdbc/qoq/functions/scalar/Trim.java | 6 +- .../jdbc/qoq/functions/scalar/Upper.java | 6 +- .../ortus/boxlang/runtime/scopes/Key.java | 2 + .../ortus/boxlang/compiler/QoQParseTest.java | 103 +++++++++++++ 56 files changed, 983 insertions(+), 103 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Avg.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Sum.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cast.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Convert.java diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 7268463b5..44878d74d 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -249,10 +249,13 @@ expr: ) expr | expr AND_ expr | expr OR_ expr + // Special handling of cast to allow cast( foo as number) + | CAST_ OPEN_PAR expr AS_ (name | STRING_LITERAL) CLOSE_PAR + // special handling of convert to allow convert( foo, number ) or convert( foo, 'number' ) + | CONVERT_ OPEN_PAR expr COMMA (name | STRING_LITERAL) CLOSE_PAR | function_name OPEN_PAR ((DISTINCT_? expr ( COMMA expr)*) | STAR)? CLOSE_PAR // filter_clause? over_clause? | OPEN_PAR expr CLOSE_PAR //| OPEN_PAR expr (COMMA expr)* CLOSE_PAR - // | CAST_ OPEN_PAR expr AS_ type_name CLOSE_PAR // | expr COLLATE_ collation_name | expr NOT_? LIKE_ expr (ESCAPE_ expr)? | expr IS_ NOT_? expr @@ -631,6 +634,7 @@ keyword: | CASCADE_ | CASE_ | CAST_ + | CONVERT_ | CHECK_ | COLLATE_ | COLUMN_ diff --git a/src/main/antlr/SQLLexer.g4 b/src/main/antlr/SQLLexer.g4 index 68b4a1162..eb754d2b2 100644 --- a/src/main/antlr/SQLLexer.g4 +++ b/src/main/antlr/SQLLexer.g4 @@ -57,6 +57,7 @@ BY_: 'BY'; CASCADE_: 'CASCADE'; CASE_: 'CASE'; CAST_: 'CAST'; +CONVERT_: 'CONVERT'; CHECK_: 'CHECK'; COLLATE_: 'COLLATE'; COLUMN_: 'COLUMN'; diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java index 9e07193a8..5ae99ed76 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.List; import java.util.Map; import java.util.Set; @@ -124,6 +125,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return QoQExec.getTableLookup().get( tableFinal ).getCell( name, rowNum - 1 ); } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return null; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + /** * Runtime check if the expression evaluates to a boolean value and works for columns as well * diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java index fcf294bcc..e38b3c3cf 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java @@ -19,11 +19,10 @@ import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.Position; -import ortus.boxlang.compiler.ast.sql.select.SQLTable; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; -import ortus.boxlang.runtime.types.Query; import ortus.boxlang.runtime.types.QueryColumnType; /** @@ -61,10 +60,21 @@ public boolean isDistinct() { /** * What type does this expression evaluate to */ - public QueryColumnType getType( Map tableLookup ) { + public QueryColumnType getType( QoQSelectExecution QoQExec ) { return QueryColumnType.INTEGER; } + /** + * Runtime check if the expression evaluates to a boolean value and works for columns as well + * + * @param QoQExec Query execution state + * + * @return true if the expression evaluates to a boolean value + */ + public boolean isBoolean( QoQSelectExecution QoQExec ) { + return false; + } + /** * Runtime check if the expression evaluates to a numeric value and works for columns as well * @@ -72,10 +82,33 @@ public QueryColumnType getType( Map tableLookup ) { * * @return true if the expression evaluates to a numeric value */ - public boolean isNumeric( Map tableLookup ) { + public boolean isNumeric( QoQSelectExecution QoQExec ) { + return true; + } + + /** + * Is function aggregate + */ + public boolean isAggregate() { return true; } + /** + * Evaluate the expression + */ + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { + throw new RuntimeException( "QoQ Function count() is an aggregate function and cannot be used in a non-aggregate context" ); + } + + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + // TODO: handle distinct + var values = buildAggregateValues( QoQExec, intersections, getArguments().get( 0 ) ); + return values.length; + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java index bf17f5a10..fde1a5a66 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLExpression.java @@ -14,6 +14,8 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.List; + import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.SQLNode; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; @@ -78,4 +80,9 @@ public QueryColumnType getType( QoQSelectExecution QoQExec ) { */ public abstract Object evaluate( QoQSelectExecution QoQExec, int[] intersection ); + /** + * Evaluate the expression aginst a partition of data + */ + public abstract Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ); + } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java index d39197b3c..56e6e26d6 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java @@ -123,7 +123,7 @@ public boolean isAggregate() { * What type does this expression evaluate to */ public QueryColumnType getType( QoQSelectExecution QoQExec ) { - return QoQFunctionService.getFunction( name ).returnType(); + return QoQFunctionService.getFunction( name ).returnType( QoQExec, arguments ); } /** @@ -133,15 +133,35 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { QoQFunction function = QoQFunctionService.getFunction( name ); if ( function.requiredParams() > arguments.size() ) { throw new RuntimeException( - "QoQ Function [" + name + "] expects at least" + function.requiredParams() + " arguments, but got " + arguments.size() ); + "QoQ Function " + name + "() expects at least" + function.requiredParams() + " arguments, but got " + arguments.size() ); } if ( function.isAggregate() ) { - return function.invokeAggregate( arguments, QoQExec ); + throw new RuntimeException( "QoQ Function " + name + "() is an aggregate function and cannot be used in a non-aggregate context" ); } else { - return function.invoke( arguments.stream().map( a -> a.evaluate( QoQExec, intersection ) ).toList() ); + return function.invoke( arguments.stream().map( a -> a.evaluate( QoQExec, intersection ) ).toList(), arguments ); } } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return null; + } + QoQFunction function = QoQFunctionService.getFunction( name ); + if ( function.isAggregate() ) { + return function.invokeAggregate( + arguments.stream().map( a -> buildAggregateValues( QoQExec, intersections, a ) ).toList(), arguments ); + } else { + return function.invoke( arguments.stream().map( a -> a.evaluateAggregate( QoQExec, intersections ) ).toList(), arguments ); + } + } + + protected Object[] buildAggregateValues( QoQSelectExecution QoQExec, List intersections, SQLExpression argument ) { + return intersections.stream().map( i -> argument.evaluate( QoQExec, i ) ).filter( v -> v != null ).toArray(); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java index ed930ad5b..6fb1794aa 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.List; import java.util.Map; import java.util.Set; @@ -120,6 +121,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return QoQExec.getSelectStatementExecution().getParams().get( index ).value(); } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return null; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java index 287bcf40c..7d7c3b9d0 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -93,6 +94,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return expression.evaluate( QoQExec, intersection ); } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + return expression.evaluateAggregate( QoQExec, intersections ); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java index 1719d9e4d..a6d827d1a 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -64,6 +65,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { throw new BoxRuntimeException( "Cannot evaluate a * expression" ); } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + throw new BoxRuntimeException( "Cannot evaluate a * expression" ); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java index 465da521f..9b9bb7a14 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.literal; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -79,6 +80,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + return value; + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java index 23be9ecb5..8dbd4c97a 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.literal; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -52,6 +53,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return null; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + return null; + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java index aadd6ad66..d5717ff7d 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.literal; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -89,6 +90,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + return value; + } + /** * Runtime check if the expression evaluates to a numeric value and works for columns as well * diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java index d93fa70f6..16f1499a9 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.literal; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -76,6 +77,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return value; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + return value; + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java index a4755f3ec..2bcb399d7 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.operation; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -135,6 +136,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return doBetween( leftValue, rightValue, expressionValue ) ^ not; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return false; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + /** * Helper for evaluating an expression as a number * diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java index 2b8ce78fc..791bfbced 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.operation; +import java.util.List; import java.util.Map; import java.util.Set; @@ -169,7 +170,7 @@ public QueryColumnType getType( QoQSelectExecution QoQExec ) { // Plus returns a string if the left and right operand were a string, otherwise it's a math operation. if ( operator == SQLBinaryOperator.PLUS ) { return QueryColumnType.isStringType( left.getType( QoQExec ) ) - && QueryColumnType.isStringType( right.getType( QoQExec ) ) + || QueryColumnType.isStringType( right.getType( QoQExec ) ) ? QueryColumnType.VARCHAR : QueryColumnType.DOUBLE; } @@ -291,6 +292,117 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + if ( isBoolean( QoQExec ) ) { + return false; + } else { + return null; + } + } + Object leftValue; + Object rightValue; + Double leftNum; + Double rightNum; + int compareResult; + // Implement each binary operator + switch ( operator ) { + case DIVIDE : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + if ( rightNum.doubleValue() == 0 ) { + throw new BoxRuntimeException( "Division by zero" ); + } + return leftNum / rightNum; + case EQUAL : + leftValue = left.evaluateAggregate( QoQExec, intersections ); + rightValue = right.evaluateAggregate( QoQExec, intersections ); + return EqualsEquals.invoke( leftValue, rightValue, true ); + case GREATERTHAN : + return Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ) == 1; + case GREATERTHANOREQUAL : + compareResult = Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ); + return compareResult == 1 || compareResult == 0; + case LESSTHAN : + return Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ) == -1; + case LESSTHANOREQUAL : + compareResult = Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ); + return compareResult == -1 || compareResult == 0; + case MINUS : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum - rightNum; + case MODULO : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum % rightNum; + case MULTIPLY : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum * rightNum; + case NOTEQUAL : + leftValue = left.evaluateAggregate( QoQExec, intersections ); + rightValue = right.evaluateAggregate( QoQExec, intersections ); + return !EqualsEquals.invoke( leftValue, rightValue, true ); + case AND : + ensureBooleanOperands( QoQExec ); + leftValue = left.evaluateAggregate( QoQExec, intersections ); + // Short circuit, don't eval right if left is false + if ( ( Boolean ) leftValue ) { + return ( Boolean ) right.evaluateAggregate( QoQExec, intersections ); + } else { + return false; + } + case OR : + ensureBooleanOperands( QoQExec ); + if ( ( Boolean ) left.evaluateAggregate( QoQExec, intersections ) ) { + return true; + } + if ( ( Boolean ) right.evaluateAggregate( QoQExec, intersections ) ) { + return true; + } + return false; + case PLUS : + if ( left.isNumeric( QoQExec ) && right.isNumeric( QoQExec ) ) { + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum + rightNum; + } else { + return Concat.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ) ); + } + case LIKE : + return doLikeAggregate( QoQExec, intersections ); + case NOTLIKE : + return !doLikeAggregate( QoQExec, intersections ); + case CONCAT : + return Concat.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ) ); + default : + throw new BoxRuntimeException( "Unknown binary operator: " + operator ); + } + } + /** * Implement LIKE so we can reuse for NOT LIKE */ @@ -304,6 +416,19 @@ private boolean doLike( QoQSelectExecution QoQExec, int[] intersection ) { return LikeOperation.invoke( leftValueStr, rightValueStr, escapeValue ); } + /** + * Implement LIKE so we can reuse for NOT LIKE + */ + private boolean doLikeAggregate( QoQSelectExecution QoQExec, List intersections ) { + String leftValueStr = StringCaster.cast( left.evaluateAggregate( QoQExec, intersections ) ); + String rightValueStr = StringCaster.cast( right.evaluateAggregate( QoQExec, intersections ) ); + String escapeValue = null; + if ( escape != null ) { + escapeValue = StringCaster.cast( escape.evaluateAggregate( QoQExec, intersections ) ); + } + return LikeOperation.invoke( leftValueStr, rightValueStr, escapeValue ); + } + /** * Reusable helper method to ensure that the left and right operands are boolean expressions or bit columns * @@ -345,6 +470,19 @@ private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExe return ( ( Number ) expression.evaluate( QoQExec, intersection ) ).doubleValue(); } + /** + * Helper for evaluating an expression as a number + * + * @param tableLookup + * @param expression + * @param i + * + * @return + */ + private double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecution QoQExec, List intersections ) { + return ( ( Number ) expression.evaluateAggregate( QoQExec, intersections ) ).doubleValue(); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java index 463d12a09..bfccf4bfa 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java @@ -119,6 +119,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return not; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return false; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java index dec7d4031..752175ddc 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.operation; +import java.util.List; import java.util.Map; import ortus.boxlang.compiler.ast.BoxNode; @@ -124,6 +125,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return not; } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return false; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + @Override public void accept( VoidBoxVisitor v ) { // TODO Auto-generated method stub diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java index efbc7f223..319b77392 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression.operation; +import java.util.List; import java.util.Map; import java.util.Set; @@ -144,6 +145,16 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } } + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( intersections.isEmpty() ) { + return null; + } + return evaluate( QoQExec, intersections.get( 0 ) ); + } + /** * Reusable helper method to ensure that the left and right operands are boolean expressions or bit columns * diff --git a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java index c31f524ae..53fccc2f6 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java +++ b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java @@ -23,7 +23,6 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BOMInputStream; @@ -146,7 +145,7 @@ protected BoxNode parserFirstStage( InputStream stream, boolean classOrInterface addErrorListeners( lexer, parser ); parser.setErrorHandler( new BoxParserErrorStrategy() ); - parser.getInterpreter().setPredictionMode( PredictionMode.SLL ); + // parser.getInterpreter().setPredictionMode( PredictionMode.SLL ); ParserRuleContext parseTree = parser.parse(); // This must run FIRST before resetting the lexer diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index bf0b0c00e..e67e63d08 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -52,6 +52,7 @@ import ortus.boxlang.parser.antlr.SQLGrammar.Table_or_subqueryContext; import ortus.boxlang.parser.antlr.SQLGrammarBaseVisitor; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; /** * This class is responsible for creating the SQL AST from the ANTLR generated parse tree. @@ -229,9 +230,17 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { where = visitExpr( ctx.whereExpr, table, joins ); } - // TODO: group by + // group by + if ( ctx.groupByExpr != null && !ctx.groupByExpr.isEmpty() ) { + SQLTable finalTable = table; + List finalJoins = joins; + groupBys = ctx.groupByExpr.stream().map( expr -> visitExpr( expr, finalTable, finalJoins ) ).toList(); + } - // TODO: having + // having + if ( ctx.havingExpr != null ) { + having = visitExpr( ctx.havingExpr, table, joins ); + } // Do this after all joins above so we know the tables available to us final SQLTable finalTable = table; @@ -239,11 +248,16 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { resultColumns = ctx.result_column().stream().map( col -> visitResult_column( col, finalTable, finalJoins ) ).toList(); var result = new SQLSelect( distinct, resultColumns, table, joins, where, groupBys, having, limit, pos, src ); + var cols = result.getDescendantsOfType( SQLColumn.class, c -> c.getTable() == null ); if ( cols.size() > 0 ) { - if ( table == null && ( joins == null || joins.isEmpty() ) ) { - tools.reportError( "This QoQ has column references, but there is no table!", pos ); - } else if ( joins == null || joins.isEmpty() ) { + if ( table == null && joins == null ) { + // Only report an error if there are no issues already + // because this can get tripped up by parsing that simply failed too soon and didn't finish + if ( tools.issues.size() == 0 ) { + tools.reportError( "This QoQ has column references, but there is no table!", pos ); + } + } else if ( joins == null ) { // If there is only one table, we know what it is now cols.forEach( c -> c.setTable( finalTable ) ); } @@ -451,6 +465,31 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.AND, pos, src, table, joins ); } else if ( ctx.OR_() != null ) { return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.OR, pos, src, table, joins ); + } else if ( ctx.CAST_() != null || ctx.CONVERT_() != null ) { + // CAST( expr AS type ) + Key functionName = ctx.CONVERT_() != null ? Key.convert : Key.cast; + List arguments = new ArrayList(); + + // Add expr as first arg + arguments.add( visitExpr( ctx.expr( 0 ), table, joins ) ); + + // Add type as second arg + // We allow both varchar or 'varchar' + SQLStringLiteral type; + if ( ctx.STRING_LITERAL() != null ) { + type = processStringLiteral( ctx.STRING_LITERAL() ); + } else { + type = new SQLStringLiteral( ctx.name().getText(), tools.getPosition( ctx.name() ), tools.getSourceText( ctx.name() ) ); + } + // validate the type here + try { + QueryColumnType.fromString( type.getValue() ); + } catch ( IllegalArgumentException e ) { + tools.reportError( "Invalid type for " + functionName.getName() + ": " + type.getValue(), pos ); + } + arguments.add( type ); + + return new SQLFunction( functionName, arguments, pos, src ); } else if ( ctx.function_name() != null ) { Key functionName = Key.of( ctx.function_name().getText() ); boolean hasDistinct = ctx.DISTINCT_() != null; @@ -560,17 +599,23 @@ public SQLExpression visitLiteral_value( Literal_valueContext ctx ) { } else if ( ctx.FALSE_() != null ) { return new SQLBooleanLiteral( false, pos, src ); } else if ( ctx.STRING_LITERAL() != null ) { - String str = ctx.STRING_LITERAL().getText(); - // strip quote chars - str = str.substring( 1, str.length() - 1 ); - // unescape `''` inside string - str = str.replace( "''", "'" ); - return new SQLStringLiteral( str, pos, src ); + return processStringLiteral( ctx.STRING_LITERAL() ); } else { throw new UnsupportedOperationException( "Unimplemented literal expression: " + src ); } } + private SQLStringLiteral processStringLiteral( TerminalNode ctx ) { + var pos = tools.getPosition( ctx ); + + String str = ctx.getText(); + // strip quote chars + str = str.substring( 1, str.length() - 1 ); + // unescape `''` inside string + str = str.replace( "''", "'" ); + return new SQLStringLiteral( str, pos, str ); + } + /** * Visit the class or interface context to generate the AST node for the * top level node diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/IQoQFunctionDef.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/IQoQFunctionDef.java index 7a6fb8534..d56e48668 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/IQoQFunctionDef.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/IQoQFunctionDef.java @@ -14,6 +14,9 @@ */ package ortus.boxlang.runtime.jdbc.qoq; +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -24,7 +27,7 @@ public interface IQoQFunctionDef { abstract public Key getName(); - abstract public QueryColumnType getReturnType(); + abstract public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ); abstract public int getMinArgs(); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java index 24ef8edb5..3ff5d650d 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQAggregateFunctionDef.java @@ -21,7 +21,7 @@ /** * I am the abstract class for QoQ function definitions */ -public abstract class QoQAggregateFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, QoQSelectExecution, Object> { +public abstract class QoQAggregateFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, List, Object> { public boolean isAggregate() { return true; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 603d7dfa0..c71ace6ba 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -16,6 +16,7 @@ import java.sql.SQLException; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -35,6 +36,7 @@ import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.ExpressionInterpreter; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.scopes.Key; @@ -142,7 +144,10 @@ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatem public static Query executeSelect( IBoxContext context, SQLSelect select, QoQStatement statement, QoQSelectStatementExecution QoQStmtExec, boolean firstSelect ) { - boolean canEarlyLimit = QoQStmtExec.getSelectStatement().getOrderBys() == null; + // If there is a group by or aggregate function, we will need to partiton the query + boolean isAggregate = select.hasAggregateResult() || select.getGroupBys() != null; + // If there is no order by, and no aggregate, we can limit the results early by stopping as soon as the limit is reached + boolean canEarlyLimit = !isAggregate && QoQStmtExec.getSelectStatement().getOrderBys() == null; Map tableLookup = new LinkedHashMap(); // This boolean expression will be used to filter the records we keep SQLExpression where = select.getWhere(); @@ -194,8 +199,9 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta // We have one or more tables, so build our stream of intersections, processing our joins as needed Stream intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); - if ( select.hasAggregateResult() ) { - + // If we have a where clause, add it as a filter to the stream + if ( where != null ) { + intersections = intersections.filter( intersection -> ( Boolean ) where.evaluate( QoQExec, intersection ) ); } // Enforce top/limit for this select. This would be a "top N" clause in the select or a "limit N" clause BEFORE the order by, which @@ -204,21 +210,18 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta intersections = intersections.limit( thisSelectLimit ); } - // If we have a where clause, add it as a filter to the stream - if ( where != null ) { - intersections = intersections.filter( intersection -> ( Boolean ) where.evaluate( QoQExec, intersection ) ); + if ( select.hasAggregateResult() || select.getGroupBys() != null ) { + return executeAggregateSelect( QoQExec, target, intersections ); } - // Process/create the rows for the final query. + // No partitioning, just create the final result set intersections.forEach( intersection -> { // System.out.println( Arrays.toString( intersection ) ); Object[] values = new Object[ resultColumns.size() ]; int colPos = 0; // Build up row data as native array for ( Key key : resultColumns.keySet() ) { - SQLResultColumn resultColumn = resultColumns.get( key ).resultColumn; - Object value = resultColumn.getExpression().evaluate( QoQExec, intersection ); - values[ colPos++ ] = value; + values[ colPos++ ] = resultColumns.get( key ).resultColumn.getExpression().evaluate( QoQExec, intersection ); } target.addRow( values ); } ); @@ -226,6 +229,55 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta return target; } + /** + * Create query partitioned by group by and aggregate functions + * + * @param qoQExec the query execution state + * @param resultColumns the result columns + * + * @return + */ + private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query target, Stream intersections ) { + Map resultColumns = QoQExec.getResultColumns(); + List groupBys = QoQExec.getSelect().getGroupBys(); + SQLExpression having = QoQExec.getSelect().getHaving(); + + // Build up our partitions + intersections.forEach( intersection -> { + String partitionKey; + if ( groupBys != null ) { + StringBuilder sb = new StringBuilder(); + for ( SQLExpression expression : groupBys ) { + // TODO: hash large values + sb.append( StringCaster.cast( expression.evaluate( QoQExec, intersection ) ) ); + } + partitionKey = sb.toString(); + } else { + partitionKey = "ALL"; + } + QoQExec.addPartition( partitionKey, intersection ); + } ); + + var partitionStream = QoQExec.getPartitions().values().stream(); + if ( QoQExec.getPartitions().size() > 50 ) { + partitionStream = partitionStream.parallel(); + } + + if ( having != null ) { + partitionStream = partitionStream.filter( partition -> ( Boolean ) having.evaluateAggregate( QoQExec, partition ) ); + } + + partitionStream.forEach( partition -> { + Object[] values = new Object[ resultColumns.size() ]; + int colPos = 0; + for ( Key key : resultColumns.keySet() ) { + values[ colPos++ ] = resultColumns.get( key ).resultColumn.getExpression().evaluateAggregate( QoQExec, partition ); + } + target.addRow( values ); + } ); + return target; + } + /** * Union two queries together, keeping all rows * diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index f14bdf597..705cb1365 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -20,14 +20,19 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Avg; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Max; +import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Min; +import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Sum; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Abs; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Acos; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Asin; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Atan; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Cast; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Ceiling; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Coalesce; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Concat; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Convert; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Cos; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Exp; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Floor; @@ -84,36 +89,45 @@ public class QoQFunctionService { // Aggregate register( Max.INSTANCE ); + register( Min.INSTANCE ); + register( Cast.INSTANCE ); + register( Convert.INSTANCE ); + register( Sum.INSTANCE ); + register( Avg.INSTANCE ); } private QoQFunctionService() { } - public static void register( Key name, java.util.function.Function, Object> function, QueryColumnType returnType, int requiredParams ) { - functions.put( name, QoQFunction.of( function, returnType, requiredParams ) ); + public static void register( Key name, java.util.function.BiFunction, List, Object> function, IQoQFunctionDef functionDef, + QueryColumnType returnType, int requiredParams ) { + functions.put( name, QoQFunction.of( function, functionDef, returnType, requiredParams ) ); } public static void registerCustom( Key name, ortus.boxlang.runtime.types.Function function, QueryColumnType returnType, int requiredParams, IBoxContext context ) { functions.put( name, QoQFunction.of( - ( List arguments ) -> context.invokeFunction( function, arguments.toArray() ), + // TODO: do we pass the expressions here? + ( List arguments, List expressions ) -> context.invokeFunction( function, arguments.toArray() ), + null, returnType, requiredParams ) ); } - public static void registerAggregate( Key name, java.util.function.BiFunction, QoQSelectExecution, Object> function, + public static void registerAggregate( Key name, java.util.function.BiFunction, List, Object> function, + IQoQFunctionDef functionDef, QueryColumnType returnType, int requiredParams ) { - functions.put( name, QoQFunction.ofAggregate( function, returnType, requiredParams ) ); + functions.put( name, QoQFunction.ofAggregate( function, functionDef, returnType, requiredParams ) ); } public static void register( QoQScalarFunctionDef functionDef ) { - register( functionDef.getName(), functionDef, functionDef.getReturnType(), functionDef.getMinArgs() ); + register( functionDef.getName(), functionDef, functionDef, null, functionDef.getMinArgs() ); } public static void register( QoQAggregateFunctionDef functionDef ) { - registerAggregate( functionDef.getName(), functionDef, functionDef.getReturnType(), functionDef.getMinArgs() ); + registerAggregate( functionDef.getName(), functionDef, functionDef, null, functionDef.getMinArgs() ); } public static void unregister( Key name ) { @@ -128,30 +142,42 @@ public static QoQFunction getFunction( Key name ) { } public record QoQFunction( - java.util.function.Function, Object> callable, - java.util.function.BiFunction, QoQSelectExecution, Object> aggregateCallable, + java.util.function.BiFunction, List, Object> callable, + java.util.function.BiFunction, List, Object> aggregateCallable, + IQoQFunctionDef functionDef, QueryColumnType returnType, int requiredParams ) { - static QoQFunction of( java.util.function.Function, Object> callable, QueryColumnType returnType, int requiredParams ) { - return new QoQFunction( callable, null, returnType, requiredParams ); + static QoQFunction of( java.util.function.BiFunction, List, Object> callable, IQoQFunctionDef functionDef, + QueryColumnType returnType, + int requiredParams ) { + return new QoQFunction( callable, null, functionDef, returnType, requiredParams ); } - static QoQFunction ofAggregate( java.util.function.BiFunction, QoQSelectExecution, Object> callable, QueryColumnType returnType, + static QoQFunction ofAggregate( java.util.function.BiFunction, List, Object> callable, IQoQFunctionDef functionDef, + QueryColumnType returnType, int requiredParams ) { - return new QoQFunction( null, callable, returnType, requiredParams ); + return new QoQFunction( null, callable, functionDef, returnType, requiredParams ); } - public Object invoke( List arguments ) { - return callable.apply( arguments ); + public Object invoke( List arguments, List expressions ) { + return callable.apply( arguments, expressions ); } - public Object invokeAggregate( List arguments, QoQSelectExecution QoQExec ) { - return aggregateCallable.apply( arguments, QoQExec ); + public Object invokeAggregate( List arguments, List expressions ) { + return aggregateCallable.apply( arguments, expressions ); } public boolean isAggregate() { return aggregateCallable != null; } + + public QueryColumnType returnType( QoQSelectExecution QoQExec, List expressions ) { + if ( functionDef != null ) { + return functionDef.getReturnType( QoQExec, expressions ); + } + return returnType; + } } + } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java index 3846e907f..555d13181 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java @@ -16,10 +16,12 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; + /** * I am the abstract class for QoQ function definitions */ -public abstract class QoQScalarFunctionDef implements IQoQFunctionDef, java.util.function.Function, Object> { +public abstract class QoQScalarFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, List, Object> { public boolean isAggregate() { return false; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index 0cddd2f19..96ffaba86 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -46,6 +46,7 @@ public class QoQSelectExecution { public Map resultColumns = null; public Map tableLookup; public QoQSelectStatementExecution selectStatementExecution; + public Map> partitions = new ConcurrentHashMap>(); private Map independentSubQueries = new ConcurrentHashMap(); @@ -201,4 +202,16 @@ public Query getIndepententSubQuery( SQLSelectStatement subquery ) { ); } + public void addPartition( String partitionName, int[] partition ) { + partitions.computeIfAbsent( partitionName, p -> new ArrayList() ).add( partition ); + } + + public List getPartition( String partitionName ) { + return partitions.get( partitionName ); + } + + public Map> getPartitions() { + return partitions; + } + } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Avg.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Avg.java new file mode 100644 index 000000000..46080548d --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Avg.java @@ -0,0 +1,56 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.aggregate; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Avg extends QoQAggregateFunctionDef { + + private static final Key name = Key.of( "avg" ); + + public static final QoQAggregateFunctionDef INSTANCE = new Avg(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + return QueryColumnType.DOUBLE; + } + + @Override + public int getMinArgs() { + return 1; + } + + @Override + public Object apply( List args, List expressions ) { + Object[] values = args.get( 0 ); + Double sum = 0D; + for ( Object value : values ) { + sum += ( ( Number ) value ).doubleValue(); + } + return sum / values.length; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java index e061c4a15..a87e2ba13 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java @@ -17,15 +17,15 @@ import java.util.List; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; -import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; public class Max extends QoQAggregateFunctionDef { - private static final Key name = Key.of( "atan" ); + private static final Key name = Key.of( "max" ); public static final QoQAggregateFunctionDef INSTANCE = new Max(); @@ -35,8 +35,9 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { - return QueryColumnType.DOUBLE; + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + // The return type of the function is the same as the type of the expression being aggregated + return expressions.get( 0 ).getType( QoQExec ); } @Override @@ -45,8 +46,17 @@ public int getMinArgs() { } @Override - public Object apply( List args, QoQSelectExecution QoQExec ) { - return ortus.boxlang.runtime.bifs.global.math.Atn._invoke( NumberCaster.cast( args.get( 0 ) ) ); + public Object apply( List args, List expressions ) { + Object[] input = args.get( 0 ); + Object max = input[ 0 ]; + if ( input.length > 1 ) { + for ( int i = 1; i < input.length; i++ ) { + if ( Compare.invoke( max, input[ i ] ) < 0 ) { + max = input[ i ]; + } + } + } + return max; } } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java new file mode 100644 index 000000000..092a51abd --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java @@ -0,0 +1,62 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.aggregate; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.operators.Compare; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Min extends QoQAggregateFunctionDef { + + private static final Key name = Key.of( "min" ); + + public static final QoQAggregateFunctionDef INSTANCE = new Min(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + // The return type of the function is the same as the type of the expression being aggregated + return expressions.get( 0 ).getType( QoQExec ); + } + + @Override + public int getMinArgs() { + return 1; + } + + @Override + public Object apply( List args, List expressions ) { + Object[] input = args.get( 0 ); + Object max = input[ 0 ]; + if ( input.length > 1 ) { + for ( int i = 1; i < input.length; i++ ) { + if ( Compare.invoke( max, input[ i ] ) > 0 ) { + max = input[ i ]; + } + } + } + return max; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Sum.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Sum.java new file mode 100644 index 000000000..8d6797e93 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Sum.java @@ -0,0 +1,56 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.aggregate; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Sum extends QoQAggregateFunctionDef { + + private static final Key name = Key.of( "sum" ); + + public static final QoQAggregateFunctionDef INSTANCE = new Sum(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + return QueryColumnType.DOUBLE; + } + + @Override + public int getMinArgs() { + return 1; + } + + @Override + public Object apply( List args, List expressions ) { + Object[] values = args.get( 0 ); + Double sum = 0D; + for ( Object value : values ) { + sum += ( ( Number ) value ).doubleValue(); + } + return sum; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Abs.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Abs.java index 3e110a5f6..b01ee97dc 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Abs.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Abs.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Abs._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Acos.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Acos.java index 1d11b03ec..bc359c010 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Acos.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Acos.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Acos._invoke( DoubleCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Asin.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Asin.java index 5fd251df3..8b939d364 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Asin.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Asin.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Asin._invoke( DoubleCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Atan.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Atan.java index 1c6c3abcc..6d187e7ea 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Atan.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Atan.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Atn._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cast.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cast.java new file mode 100644 index 000000000..28084148f --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cast.java @@ -0,0 +1,60 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLStringLiteral; +import ortus.boxlang.runtime.BoxRuntime; +import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.GenericCaster; +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Cast extends QoQScalarFunctionDef { + + private static final Key name = Key.of( "cast" ); + + public static final QoQScalarFunctionDef INSTANCE = new Cast(); + + private static final IBoxContext runtimeContext = BoxRuntime.getInstance().getRuntimeContext(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + // Parser forces the second arg to always be a string literal + return QueryColumnType.fromString( ( ( SQLStringLiteral ) expressions.get( 1 ) ).getValue() ); + } + + @Override + public int getMinArgs() { + return 2; + } + + @Override + public Object apply( List args, List expressions ) { + Object value = args.get( 0 ); + String type = QueryColumnType.fromString( ( ( SQLStringLiteral ) expressions.get( 1 ) ).getValue() ).toString(); + return GenericCaster.cast( runtimeContext, value, type ); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ceiling.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ceiling.java index 92e182cac..8943e8958 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ceiling.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ceiling.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Ceiling._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Coalesce.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Coalesce.java index a19260909..ca7d15733 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Coalesce.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Coalesce.java @@ -16,7 +16,9 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -32,7 +34,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.OBJECT; } @@ -42,7 +44,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { for ( Object arg : args ) { if ( arg != null ) { return arg; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Concat.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Concat.java index 1bffe6b3e..c2f28b322 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Concat.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Concat.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { StringBuilder sb = new StringBuilder(); for ( Object arg : args ) { sb.append( StringCaster.cast( arg ) ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Convert.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Convert.java new file mode 100644 index 000000000..5a44d21d8 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Convert.java @@ -0,0 +1,31 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.scopes.Key; + +public class Convert extends Cast { + + private static final Key name = Key.of( "convert" ); + + public static final QoQScalarFunctionDef INSTANCE = new Convert(); + + @Override + public Key getName() { + return name; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cos.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cos.java index 022c22893..1efc54a49 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cos.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Cos.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Cos._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Exp.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Exp.java index 4e42a75f3..4a9726d29 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Exp.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Exp.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Exp._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Floor.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Floor.java index 099ea4733..9eb69db35 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Floor.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Floor.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Floor._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java index 8ff5370c7..79ac0aa1e 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java @@ -16,7 +16,9 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -32,7 +34,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.OBJECT; } @@ -42,7 +44,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return args.get( 0 ) == null; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java index 94ec35635..f958e090d 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.INTEGER; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return StringCaster.cast( args.get( 0 ) ).length(); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java index e03438d2e..180e09f48 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return StringCaster.cast( args.get( 0 ) ).toLowerCase(); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ltrim.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ltrim.java index 5c347942a..f59b0991b 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ltrim.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Ltrim.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { String str = StringCaster.cast( args.get( 0 ) ); int start = 0; while ( start < str.length() && Character.isWhitespace( str.charAt( start ) ) ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java index 18fa4553c..15cc2c3a7 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java @@ -16,7 +16,9 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.operators.Modulus; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { Object left = args.get( 0 ); Object right = args.get( 1 ); if ( left == null || right == null ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java index ba2a335df..5662ccb2f 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { Object left = args.get( 0 ); Object right = args.get( 1 ); if ( left == null || right == null ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Rtrim.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Rtrim.java index 596dd2987..cb948bc64 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Rtrim.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Rtrim.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { String str = StringCaster.cast( args.get( 0 ) ); int end = str.length(); while ( end > 0 && Character.isWhitespace( str.charAt( end - 1 ) ) ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sin.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sin.java index f507eec43..6395312e4 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sin.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sin.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Sin._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sqrt.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sqrt.java index 7cb81235a..523cafec2 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sqrt.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Sqrt.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Sqr._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Tan.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Tan.java index 5f58f3c42..be289a66a 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Tan.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Tan.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.DOUBLE; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return ortus.boxlang.runtime.bifs.global.math.Tan._invoke( NumberCaster.cast( args.get( 0 ) ) ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java index 81bc38e17..100602ed9 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return StringCaster.cast( args.get( 0 ) ).trim(); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java index a2742afa6..5fcab6449 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java @@ -16,8 +16,10 @@ import java.util.List; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -33,7 +35,7 @@ public Key getName() { } @Override - public QueryColumnType getReturnType() { + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { return QueryColumnType.VARCHAR; } @@ -43,7 +45,7 @@ public int getMinArgs() { } @Override - public Object apply( List args ) { + public Object apply( List args, List expressions ) { return StringCaster.cast( args.get( 0 ) ).toUpperCase(); } diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java index 7e51bfaa3..9adedd9bf 100644 --- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java +++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java @@ -149,6 +149,7 @@ public class Key implements Comparable, Serializable { public static final Key caseSensitive = Key.of( "caseSensitive" ); public static final Key category = Key.of( "category" ); public static final Key cause = Key.of( "cause" ); + public static final Key cast = Key.of( "cast" ); public static final Key cert_cookie = Key.of( "cert_cookie" ); public static final Key cert_flags = Key.of( "cert_flags" ); public static final Key cert_issuer = Key.of( "cert_issuer" ); @@ -198,6 +199,7 @@ public class Key implements Comparable, Serializable { public static final Key context = Key.of( "context" ); public static final Key context_path = Key.of( "context_path" ); public static final Key contextual = Key.of( "contextual" ); + public static final Key convert = Key.of( "convert" ); public static final Key conversionType = Key.of( "conversionType" ); public static final Key cookies = Key.of( "cookies" ); public static final Key copy = Key.of( "copy" ); diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index e8821c69f..bf8dc7933 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -30,6 +30,8 @@ import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.VariablesScope; import ortus.boxlang.runtime.types.IStruct; +import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.Struct; public class QoQParseTest { @@ -229,4 +231,105 @@ select reverse( 'Brad' ) as rev context ); } + @Test + public void testAggregate() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select count( 1 ) count, + max(age) maxAge, + min(age) minAge, + min(age+0)+1 minAgePlusOne, + concat( 'foo', cast( min(age) as string)) aggregateInScalar, + concat( 'foo', cast( max(age) as string)) aggregateInScalar2, + sum( age ) sumAge, + avg(age) avgAge + from qryEmployees + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + + @Test + public void testCast() { + instance.executeSource( + """ + q = queryExecute( " + select cast( 5 as string) + 4 as result, 5 as result2, cast( 5 as 'string') as result3 + ", + [], + { dbType : "query" } + ); + println( q ) + result = q + + """, + context ); + Query q = variables.getAsQuery( result ); + assertThat( q.getColumn( result ).getType() ).isEqualTo( QueryColumnType.VARCHAR ); + assertThat( q.getColumn( Key.of( "result2" ) ).getType() ).isEqualTo( QueryColumnType.DOUBLE ); + assertThat( q.getColumn( Key.of( "result3" ) ).getType() ).isEqualTo( QueryColumnType.VARCHAR ); + instance.executeSource( + """ + q = queryExecute( " + select convert( 5, string) + 4 as result, 5 as result2, convert( 5, 'string') as result3 + ", + [], + { dbType : "query" } + ); + println( q ) + result = q + + """, + context ); + q = variables.getAsQuery( result ); + assertThat( q.getColumn( result ).getType() ).isEqualTo( QueryColumnType.VARCHAR ); + assertThat( q.getColumn( Key.of( "result2" ) ).getType() ).isEqualTo( QueryColumnType.DOUBLE ); + assertThat( q.getColumn( Key.of( "result3" ) ).getType() ).isEqualTo( QueryColumnType.VARCHAR ); + } + + @Test + public void testAggregateGroup() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select upper( dept) as dept, count(1), max(name), min(name) + from qryEmployees as t + group by dept + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + } From 1def498eec777264d34865e0ac09d8cbbc386f1e Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Tue, 17 Dec 2024 16:53:57 -0600 Subject: [PATCH 022/161] BL-823 --- .../runtime/jdbc/qoq/QoQFunctionService.java | 4 ++ .../qoq/functions/aggregate/GroupConcat.java | 64 +++++++++++++++++++ .../qoq/functions/aggregate/StringAgg.java | 31 +++++++++ .../ortus/boxlang/compiler/QoQParseTest.java | 2 +- 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/GroupConcat.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/StringAgg.java diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index 705cb1365..51863e8ca 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -21,8 +21,10 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Avg; +import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.GroupConcat; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Max; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Min; +import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.StringAgg; import ortus.boxlang.runtime.jdbc.qoq.functions.aggregate.Sum; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Abs; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Acos; @@ -94,6 +96,8 @@ public class QoQFunctionService { register( Convert.INSTANCE ); register( Sum.INSTANCE ); register( Avg.INSTANCE ); + register( GroupConcat.INSTANCE ); + register( StringAgg.INSTANCE ); } private QoQFunctionService() { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/GroupConcat.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/GroupConcat.java new file mode 100644 index 000000000..1f947ad8a --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/GroupConcat.java @@ -0,0 +1,64 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.aggregate; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; +import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class GroupConcat extends QoQAggregateFunctionDef { + + private static final Key name = Key.of( "group_concat" ); + + public static final QoQAggregateFunctionDef INSTANCE = new GroupConcat(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + return QueryColumnType.VARCHAR; + } + + @Override + public int getMinArgs() { + return 1; + } + + @Override + public Object apply( List args, List expressions ) { + Object[] values = args.get( 0 ); + String separator = ","; + if ( args.size() > 1 ) { + separator = ( String ) args.get( 1 )[ 0 ]; + } + StringBuilder sb = new StringBuilder(); + for ( Object value : values ) { + if ( sb.length() > 0 ) { + sb.append( separator ); + } + sb.append( StringCaster.cast( value ) ); + } + return sb.toString(); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/StringAgg.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/StringAgg.java new file mode 100644 index 000000000..be7608dd2 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/StringAgg.java @@ -0,0 +1,31 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.aggregate; + +import ortus.boxlang.runtime.jdbc.qoq.QoQAggregateFunctionDef; +import ortus.boxlang.runtime.scopes.Key; + +public class StringAgg extends GroupConcat { + + private static final Key name = Key.of( "string_agg" ); + + public static final QoQAggregateFunctionDef INSTANCE = new StringAgg(); + + @Override + public Key getName() { + return name; + } + +} diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index bf8dc7933..4797216b6 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -319,7 +319,7 @@ public void testAggregateGroup() { ) q = queryExecute( " - select upper( dept) as dept, count(1), max(name), min(name) + select upper( dept) as dept, count(1), max(name), min(name), GROUP_CONCAT( name) as names, GROUP_CONCAT( name, ' | ') as namesPipe from qryEmployees as t group by dept ", From 74885d71e41b5fffebf3b1326a89ae50ea7423e9 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Tue, 17 Dec 2024 17:12:49 -0600 Subject: [PATCH 023/161] test having --- .../ortus/boxlang/compiler/QoQParseTest.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 4797216b6..dad35ed32 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -307,28 +307,30 @@ select convert( 5, string) + 4 as result, 5 as result2, convert( 5, 'string') as public void testAggregateGroup() { instance.executeSource( """ - qryEmployees = queryNew( - "name,age,dept,supervisor", - "varchar,integer,varchar,varchar", - [ - ["luis",43,"Exec","luis"], - ["brad",44,"IT","luis"], - ["jacob",35,"IT","luis"], - ["Jon",45,"HR","luis"] - ] - ) - - q = queryExecute( " - select upper( dept) as dept, count(1), max(name), min(name), GROUP_CONCAT( name) as names, GROUP_CONCAT( name, ' | ') as namesPipe - from qryEmployees as t - group by dept - ", - [], - { dbType : "query" } - ); - println( q ) - - """, + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select upper( dept) as dept, count(1), max(name), min(name), GROUP_CONCAT( name) as names, GROUP_CONCAT( name, ' | ') as namesPipe + from qryEmployees as t + group by dept + having (count(1)+1) > 1 + order by count(1) desc + ", + [], + { dbType : "query" } + ); + println( q ) + + """, context ); } From f0a3c06b5180bde71f3425f7f2f05121a65ee477 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Tue, 17 Dec 2024 22:05:56 -0600 Subject: [PATCH 024/161] BL-823 --- .../compiler/toolchain/SQLVisitor.java | 30 ++++++++++---- .../runtime/jdbc/qoq/QoQExecutionService.java | 41 +++++++++++++++---- .../ortus/boxlang/compiler/QoQParseTest.java | 39 ++++++++++++++---- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index e67e63d08..ad4d6f57a 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -339,15 +339,15 @@ public SQLTableVariable visitTable( TableContext ctx ) { var pos = tools.getPosition( ctx ); var src = tools.getSourceText( ctx ); String schema = null; - String name = ctx.table_name().getText(); + String name = unwrapBracket( ctx.table_name().getText() ); String alias = null; if ( ctx.schema_name() != null ) { - schema = ctx.schema_name().getText(); + schema = unwrapBracket( ctx.schema_name().getText() ); } if ( ctx.table_alias() != null ) { - alias = ctx.table_alias().getText(); + alias = unwrapBracket( ctx.table_alias().getText() ); } return new SQLTableVariable( schema, name, alias, tableIndex++, pos, src ); @@ -385,7 +385,7 @@ public SQLResultColumn visitResult_column( Result_columnContext ctx, SQLTable ta SQLExpression expression; if ( ctx.column_alias() != null ) { - alias = ctx.column_alias().getText(); + alias = unwrapBracket( ctx.column_alias().getText() ); } if ( ctx.STAR() != null ) { @@ -443,14 +443,14 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j SQLTable tableRef = null; // if we have tableName.* or tAlias.* then we need to find the table reference if ( ctx.table_name() != null ) { - String tableName = ctx.table_name().getText(); + String tableName = unwrapBracket( ctx.table_name().getText() ); tableRef = findTableRef( table, joins, tableName ); // If we didn't find the table reference then error if ( tableRef == null ) { tools.reportError( "Table reference not found for " + src, pos ); } } - return new SQLColumn( tableRef, ctx.column_name().getText(), pos, src ); + return new SQLColumn( tableRef, unwrapBracket( ctx.column_name().getText() ), pos, src ); } else if ( ctx.literal_value() != null ) { return ( SQLExpression ) visit( ctx.literal_value() ); } else if ( ctx.EQ() != null || ctx.ASSIGN() != null || ctx.IS_() != null ) { @@ -479,7 +479,7 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j if ( ctx.STRING_LITERAL() != null ) { type = processStringLiteral( ctx.STRING_LITERAL() ); } else { - type = new SQLStringLiteral( ctx.name().getText(), tools.getPosition( ctx.name() ), tools.getSourceText( ctx.name() ) ); + type = new SQLStringLiteral( unwrapBracket( ctx.name().getText() ), tools.getPosition( ctx.name() ), tools.getSourceText( ctx.name() ) ); } // validate the type here try { @@ -491,7 +491,7 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j return new SQLFunction( functionName, arguments, pos, src ); } else if ( ctx.function_name() != null ) { - Key functionName = Key.of( ctx.function_name().getText() ); + Key functionName = Key.of( unwrapBracket( ctx.function_name().getText() ) ); boolean hasDistinct = ctx.DISTINCT_() != null; List arguments = new ArrayList(); if ( ctx.STAR() != null ) { @@ -572,6 +572,20 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j } } + /** + * Unwrap the brackets from the table or column name + * + * @param text the text to unwrap + * + * @return the unwrapped text + */ + private String unwrapBracket( String text ) { + if ( text.startsWith( "[" ) && text.endsWith( "]" ) ) { + return text.substring( 1, text.length() - 1 ); + } + return text; + } + private SQLExpression binarySimple( ExprContext left, ExprContext right, SQLBinaryOperator op, Position pos, String src, SQLTable table, List joins ) { return new SQLBinaryOperation( visitExpr( left, table, joins ), visitExpr( right, table, joins ), op, pos, src ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index c71ace6ba..10caba724 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -15,9 +15,11 @@ package ortus.boxlang.runtime.jdbc.qoq; import java.sql.SQLException; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import ortus.boxlang.compiler.ast.sql.SQLNode; @@ -91,14 +93,23 @@ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatem Query target = executeSelect( context, selectStatement.getSelect(), statement, QoQStmtExec, true ); if ( selectStatement.getUnions() != null ) { + + // We actually only need to de-dupe the last union, so we need to find it + // This is a performance optimization to avoid de-duping every union uneccessarily + int lastUnion = Stream.iterate( 0, i -> i + 1 ) + .limit( selectStatement.getUnions().size() ) + .filter( i -> selectStatement.getUnions().get( i ).getType() == SQLUnionType.DISTINCT ) + .reduce( ( first, second ) -> second ) + .orElse( -1 ); + + int i = 0; for ( SQLUnion union : selectStatement.getUnions() ) { Query unionQuery = executeSelect( context, union.getSelect(), statement, QoQStmtExec, false ); - if ( union.getType() == SQLUnionType.ALL ) { - unionAll( target, unionQuery ); - } else { - // distinct - unionDistinct( target, unionQuery ); + unionAll( target, unionQuery ); + if ( i == lastUnion ) { + deDupeQuery( target ); } + i++; } } @@ -296,9 +307,23 @@ private static void unionAll( Query target, Query unionQuery ) { * @param target the target query * @param unionQuery the query to union */ - private static void unionDistinct( Query target, Query unionQuery ) { - // TODO: IMPLEMENT! - unionAll( target, unionQuery ); + private static void deDupeQuery( Query target ) { + Set seen = new HashSet<>(); + // loop over rows, build partition key out of all values + for ( int i = 0; i < target.size(); i++ ) { + StringBuilder sb = new StringBuilder(); + Object[] row = target.getRow( i ); + for ( Object value : row ) { + sb.append( value ); + } + String key = sb.toString(); + if ( !seen.contains( key ) ) { + seen.add( key ); + } else { + target.deleteRow( i ); + i--; + } + } } /** diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index dad35ed32..8f2c68c1d 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -40,6 +40,7 @@ public class QoQParseTest { IBoxContext context; IScope variables; static Key result = new Key( "result" ); + static Key q = new Key( "q" ); @BeforeAll public static void setUp() { @@ -95,8 +96,8 @@ public void testRunQoQ() { ) qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200],["Janitor",200]] ) q = queryExecute( " - select e.*, s.name as supName, d.name as deptname - from qryEmployees e + select e.*, [s].[name] as [supName], d.name as deptname + from [variables].[qryEmployees] e inner join qryEmployees s on e.supervisor = s.name full join qryDept d on e.dept = d.name where d.name in ('IT','HR') @@ -128,6 +129,30 @@ public void testRunQoQUnion() { context ); } + @Test + public void testRunQoQUnionDistinct() { + instance.executeSource( + """ + q = queryExecute( " + select 'foo' as col + union select 'foo' + union select 'foo' + union select 'foo' + union select 'foo' + union select 'foo' + union select 'foo' -- Actual de-duplication runs here + union all select 'foo' + union all select 'foo' + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + assertThat( variables.getAsQuery( q ).size() ).isEqualTo( 3 ); + } + @Test public void testSubquery() { instance.executeSource( @@ -247,14 +272,14 @@ public void testAggregate() { q = queryExecute( " select count( 1 ) count, - max(age) maxAge, - min(age) minAge, + [max](age) maxAge, + min([e].[age]) minAge, min(age+0)+1 minAgePlusOne, - concat( 'foo', cast( min(age) as string)) aggregateInScalar, + concat( 'foo', cast( min(age) as [string])) aggregateInScalar, concat( 'foo', cast( max(age) as string)) aggregateInScalar2, sum( age ) sumAge, avg(age) avgAge - from qryEmployees + from qryEmployees [e] ", [], { dbType : "query" } @@ -287,7 +312,7 @@ select cast( 5 as string) + 4 as result, 5 as result2, cast( 5 as 'string') as r instance.executeSource( """ q = queryExecute( " - select convert( 5, string) + 4 as result, 5 as result2, convert( 5, 'string') as result3 + select convert( 5, [string]) + 4 as result, 5 as result2, convert( 5, 'string') as result3 ", [], { dbType : "query" } From 3a509a2719d95c11897cb5be5efa54557931a4bb Mon Sep 17 00:00:00 2001 From: Jacob Beers Date: Tue, 17 Dec 2024 22:40:18 -0600 Subject: [PATCH 025/161] BL-848 fix class locator issues --- .../boxlang/compiler/asmboxpiler/AsmTranspiler.java | 10 ++++++---- .../expression/BoxFunctionInvocationTransformer.java | 3 +-- src/test/java/TestCases/phase3/ClassTest.java | 1 - 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java index 0ea8e400c..5ced452e5 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java @@ -729,17 +729,19 @@ public List> transformProperties( Type declaringType, Lis } else { init = List.of( new InsnNode( Opcodes.ACONST_NULL ) ); - Type type = Type.getType( "L" + getProperty( "packageName" ).replace( '.', '/' ) + Type type = Type.getType( "L" + getProperty( "packageName" ).replace( '.', '/' ) + "/" + getProperty( "classname" ) + "$Lambda_" + incrementAndGetLambdaCounter() + ";" ); - List body = transform( defaultAnnotation.getValue(), TransformerContext.NONE, ReturnValueContext.VALUE_OR_NULL ); - ClassNode classNode = new ClassNode(); + ClassNode classNode = new ClassNode(); AsmHelper.init( classNode, false, type, Type.getType( Object.class ), methodVisitor -> { }, Type.getType( DefaultExpression.class ) ); AsmHelper.methodWithContextAndClassLocator( classNode, "evaluate", Type.getType( IBoxContext.class ), Type.getType( Object.class ), false, this, false, - () -> body ); + () -> { + List body = transform( defaultAnnotation.getValue(), TransformerContext.NONE, ReturnValueContext.VALUE_OR_NULL ); + return body; + } ); setAuxiliary( type.getClassName(), classNode ); initLambda = List.of( diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxFunctionInvocationTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxFunctionInvocationTransformer.java index e5639c5fc..2d7baa074 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxFunctionInvocationTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxFunctionInvocationTransformer.java @@ -21,7 +21,6 @@ import org.objectweb.asm.Type; import org.objectweb.asm.tree.AbstractInsnNode; import org.objectweb.asm.tree.InsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.AsmTranspiler; @@ -44,7 +43,7 @@ public List transform( BoxNode node, TransformerContext contex boolean safe = function.getName().equalsIgnoreCase( "isnull" ) ? true : false; List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); TransformerContext argContext = safe ? TransformerContext.SAFE : context; nodes.addAll( AsmHelper.callinvokeFunction( transpiler, Type.getType( Key.class ), function.getArguments(), transpiler.createKey( function.getName() ), diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 5450c41f6..458cbcc7c 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1558,7 +1558,6 @@ public void testMixinsPublic() { @DisplayName( "class locator usage in static initializer" ) @Test - @Disabled( "BL-848" ) public void testClassLocatorInStaticInitializer() { instance.executeSource( """ From da5bbbe91c620edb4d44fe9402ca67925994142c Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 00:02:29 -0600 Subject: [PATCH 026/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 10 +- .../ast/sql/select/expression/SQLCase.java | 250 ++++++++++++++++++ .../select/expression/SQLCaseWhenThen.java | 98 +++++++ .../compiler/toolchain/SQLVisitor.java | 30 +++ .../runtime/jdbc/qoq/QoQExecutionService.java | 35 +++ .../runtime/jdbc/qoq/QoQFunctionService.java | 100 +++++++ .../jdbc/qoq/QoQIntersectionGenerator.java | 40 +++ .../jdbc/qoq/QoQScalarFunctionDef.java | 3 + .../runtime/jdbc/qoq/QoQSelectExecution.java | 64 +++++ .../jdbc/qoq/QoQSelectStatementExecution.java | 74 ++++++ .../ortus/boxlang/compiler/QoQParseTest.java | 134 ++++++++++ 11 files changed, 837 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java create mode 100644 src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 44878d74d..63a5e72be 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -268,10 +268,18 @@ expr: // | (schema_name DOT)? table_function_name OPEN_PAR (expr (COMMA expr)*)? CLOSE_PAR ) // | ((NOT_)? EXISTS_)? OPEN_PAR select_stmt CLOSE_PAR - // | CASE_ expr? (WHEN_ expr THEN_ expr)+ (ELSE_ expr)? END_ + | case_expr // | raise_function ; +case_expr: + CASE_ initial_expr = expr? case_when_then+ (ELSE_ else_expr = expr)? END_ +; + +case_when_then: + WHEN_ when_expr = expr THEN_ then_expr = expr +; + raise_function: RAISE_ OPEN_PAR (IGNORE_ | (ROLLBACK_ | ABORT_ | FAIL_) COMMA error_message) CLOSE_PAR ; diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java new file mode 100644 index 000000000..25571fce0 --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java @@ -0,0 +1,250 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select.expression; + +import java.util.List; +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.operators.Compare; +import ortus.boxlang.runtime.types.QueryColumnType; +import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; + +/** + * Abstract Node class representing SQL case expression + */ +public class SQLCase extends SQLExpression { + + private SQLExpression inputExpression; + private List whenThens; + private SQLExpression elseExpression; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLCase( SQLExpression inputExpression, List whenThens, SQLExpression elseExpression, Position position, String sourceText ) { + super( position, sourceText ); + setInputExpression( inputExpression ); + setWhenThens( whenThens ); + setElseExpression( elseExpression ); + } + + /** + * Get the input expression + */ + public SQLExpression getInputExpression() { + return inputExpression; + } + + /** + * Set the input expression + */ + public void setInputExpression( SQLExpression inputExpression ) { + replaceChildren( this.inputExpression, inputExpression ); + this.inputExpression = inputExpression; + if ( this.inputExpression != null ) { + this.inputExpression.setParent( this ); + } + } + + /** + * Get the when clauses + */ + public List getWhenThens() { + return whenThens; + } + + /** + * Set the when clauses + */ + public void setWhenThens( List whenThens ) { + replaceChildren( this.whenThens, whenThens ); + this.whenThens = whenThens; + this.whenThens.forEach( e -> e.setParent( this ) ); + } + + /** + * Get the else expression + */ + public SQLExpression getElseExpression() { + return elseExpression; + } + + /** + * Set the else expression + */ + public void setElseExpression( SQLExpression elseExpression ) { + replaceChildren( this.elseExpression, elseExpression ); + this.elseExpression = elseExpression; + if ( this.elseExpression != null ) { + this.elseExpression.setParent( this ); + } + } + + /** + * Runtime check if the expression evaluates to a boolean value and works for columns as well + * + * @param QoQExec Query execution state + * + * @return true if the expression evaluates to a boolean value + */ + public boolean isBoolean( QoQSelectExecution QoQExec ) { + // Check first then and guess the rest are the same + return whenThens.get( 0 ).getThenExpression().isBoolean( QoQExec ); + } + + /** + * Runtime check if the expression evaluates to a numeric value and works for columns as well + * + * @param QoQExec Query execution state + * + * @return true if the expression evaluates to a numeric value + */ + public boolean isNumeric( QoQSelectExecution QoQExec ) { + // Check first then and guess the rest are the same + return whenThens.get( 0 ).getThenExpression().isNumeric( QoQExec ); + } + + /** + * What type does this expression evaluate to + */ + public QueryColumnType getType( QoQSelectExecution QoQExec ) { + // Check first then and guess the rest are the same + return whenThens.get( 0 ).getThenExpression().getType( QoQExec ); + } + + /** + * Evaluate the expression + */ + public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { + if ( inputExpression == null ) { + return processStandardCase( QoQExec, intersection, null ); + } else { + return processInputCase( QoQExec, intersection, null ); + } + } + + /** + * Evaluate the expression aginst a partition of data + */ + public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { + if ( inputExpression == null ) { + return processStandardCase( QoQExec, null, intersections ); + } else { + return processInputCase( QoQExec, null, intersections ); + } + } + + /** + * Process case/when/then with no input expression. Each when expression must be a boolean + * + * @param QoQExec Query execution state + * @param intersection The intersection of the data to evaluate + * @param intersections The list of intersections to evaluate + * + * @return The result of the case expression + */ + private Object processStandardCase( QoQSelectExecution QoQExec, int[] intersection, List intersections ) { + + // If any when case when expressions are not boolean throw an error + for ( SQLCaseWhenThen whenThen : whenThens ) { + if ( !whenThen.getWhenExpression().isBoolean( QoQExec ) ) { + throw new BoxRuntimeException( "Case/When/Then expressions must be boolean. The case [" + whenThen.getWhenExpression().getSourceText() + + "] is a [" + whenThen.getWhenExpression().getType( QoQExec ).toString() + "]" ); + } + } + for ( SQLCaseWhenThen whenThen : whenThens ) { + Boolean result; + if ( intersection != null ) { + result = ( Boolean ) whenThen.getWhenExpression().evaluate( QoQExec, intersection ); + } else { + result = ( Boolean ) whenThen.getWhenExpression().evaluateAggregate( QoQExec, intersections ); + } + if ( result ) { + return whenThen.getThenExpression().evaluate( QoQExec, intersection ); + } + } + if ( elseExpression != null ) { + return elseExpression.evaluate( QoQExec, intersection ); + } + return null; + } + + private Object processInputCase( QoQSelectExecution QoQExec, int[] intersection, List intersections ) { + Object inputValue; + if ( intersection != null ) { + inputValue = inputExpression.evaluate( QoQExec, intersection ); + } else { + inputValue = inputExpression.evaluateAggregate( QoQExec, intersections ); + } + + for ( SQLCaseWhenThen whenThen : whenThens ) { + Boolean result; + Object caseValue; + if ( intersection != null ) { + caseValue = whenThen.getWhenExpression().evaluate( QoQExec, intersection ); + } else { + caseValue = whenThen.getWhenExpression().evaluateAggregate( QoQExec, intersections ); + } + if ( Compare.invoke( inputValue, caseValue ) == 0 ) { + return whenThen.getThenExpression().evaluate( QoQExec, intersection ); + } + } + if ( elseExpression != null ) { + return elseExpression.evaluate( QoQExec, intersection ); + } + return null; + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + if ( inputExpression != null ) { + map.put( "inputExpression", inputExpression.toMap() ); + } else { + map.put( "inputExpression", null ); + } + + map.put( "whenThens", whenThens.stream().map( SQLCaseWhenThen::toMap ).toArray() ); + + if ( elseExpression != null ) { + map.put( "elseExpression", elseExpression.toMap() ); + } else { + map.put( "elseExpression", null ); + } + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java new file mode 100644 index 000000000..8f59375ef --- /dev/null +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java @@ -0,0 +1,98 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.compiler.ast.sql.select.expression; + +import java.util.Map; + +import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.Position; +import ortus.boxlang.compiler.ast.sql.SQLNode; +import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; +import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; + +/** + * Abstract Node class representing SQL case when/then expression + */ +public class SQLCaseWhenThen extends SQLNode { + + private SQLExpression whenExpression; + private SQLExpression thenExpression; + + /** + * Constructor + * + * @param position position of the statement in the source code + * @param sourceText source code of the statement + */ + public SQLCaseWhenThen( SQLExpression whenExpression, SQLExpression thenExpression, Position position, String sourceText ) { + super( position, sourceText ); + setWhenExpression( whenExpression ); + setThenExpression( thenExpression ); + } + + /** + * Get the when expression + */ + public SQLExpression getWhenExpression() { + return whenExpression; + } + + /** + * Set the when expression + */ + public void setWhenExpression( SQLExpression whenExpression ) { + replaceChildren( this.whenExpression, whenExpression ); + this.whenExpression = whenExpression; + this.whenExpression.setParent( this ); + } + + /** + * Get the then expression + */ + public SQLExpression getThenExpression() { + return thenExpression; + } + + /** + * Set the then expression + */ + public void setThenExpression( SQLExpression thenExpression ) { + replaceChildren( this.thenExpression, thenExpression ); + this.thenExpression = thenExpression; + this.thenExpression.setParent( this ); + } + + @Override + public void accept( VoidBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public BoxNode accept( ReplacingBoxVisitor v ) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + } + + @Override + public Map toMap() { + Map map = super.toMap(); + + map.put( "whenExpression", whenExpression.toMap() ); + map.put( "thenExpression", thenExpression.toMap() ); + return map; + } + +} diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index ad4d6f57a..60d92efda 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -17,6 +17,8 @@ import ortus.boxlang.compiler.ast.sql.select.SQLTableVariable; import ortus.boxlang.compiler.ast.sql.select.SQLUnion; import ortus.boxlang.compiler.ast.sql.select.SQLUnionType; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCase; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCaseWhenThen; import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; import ortus.boxlang.compiler.ast.sql.select.expression.SQLCountFunction; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; @@ -38,6 +40,7 @@ import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperator; import ortus.boxlang.compiler.parser.SQLParser; import ortus.boxlang.parser.antlr.SQLGrammar; +import ortus.boxlang.parser.antlr.SQLGrammar.Case_exprContext; import ortus.boxlang.parser.antlr.SQLGrammar.ExprContext; import ortus.boxlang.parser.antlr.SQLGrammar.Literal_valueContext; import ortus.boxlang.parser.antlr.SQLGrammar.Ordering_termContext; @@ -555,6 +558,8 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j } else if ( ctx.OPEN_PAR() != null ) { // Needs to run AFTER function and IN checks return new SQLParenthesis( visitExpr( ctx.expr( 0 ), table, joins ), pos, src ); + } else if ( ctx.case_expr() != null ) { + return visitCase( ctx.case_expr(), table, joins ); } else if ( ctx.unary_operator() != null ) { SQLUnaryOperator op; if ( ctx.unary_operator().BANG() != null ) { @@ -572,6 +577,31 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j } } + private SQLExpression visitCase( Case_exprContext ctx, SQLTable table, List joins ) { + var pos = tools.getPosition( ctx ); + var src = tools.getSourceText( ctx ); + + SQLExpression inputExpression = null; + List whenThens = new ArrayList(); + SQLExpression elseExpression = null; + + if ( ctx.initial_expr != null ) { + inputExpression = visitExpr( ctx.initial_expr, table, joins ); + } + + if ( ctx.else_expr != null ) { + elseExpression = visitExpr( ctx.else_expr, table, joins ); + } + + for ( var whenThenCtx : ctx.case_when_then() ) { + SQLExpression when = visitExpr( whenThenCtx.when_expr, table, joins ); + SQLExpression then = visitExpr( whenThenCtx.then_expr, table, joins ); + whenThens.add( new SQLCaseWhenThen( when, then, tools.getPosition( whenThenCtx ), tools.getSourceText( whenThenCtx ) ) ); + } + + return new SQLCase( inputExpression, whenThens, elseExpression, pos, src ); + } + /** * Unwrap the brackets from the table or column name * diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 10caba724..205f63c02 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -60,6 +60,13 @@ public class QoQExecutionService { */ private static final FRTransService frTransService = FRTransService.getInstance( true ); + /** + * Parse a SQL string into an AST + * + * @param sql the SQL string + * + * @return the AST + */ public static SQLNode parseSQL( String sql ) { DynamicObject trans = frTransService.startTransaction( "BL QoQ Parse", "" ); SQLParser parser = new SQLParser(); @@ -83,6 +90,15 @@ public static SQLNode parseSQL( String sql ) { return ( SQLNode ) result.getRoot(); } + /** + * Execute a QoQ statement + * + * @param context the context + * @param selectStatement the select statement + * @param statement the QoQ statement + * + * @return the query + */ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatement selectStatement, QoQStatement statement ) { QoQSelectStatementExecution QoQStmtExec = QoQSelectStatementExecution.of( @@ -153,6 +169,17 @@ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatem return target; } + /** + * Execute a select statement + * + * @param context the context + * @param select the select + * @param statement the QoQ statement + * @param QoQStmtExec the QoQ statement execution + * @param firstSelect if this is the first select + * + * @return the query + */ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQStatement statement, QoQSelectStatementExecution QoQStmtExec, boolean firstSelect ) { // If there is a group by or aggregate function, we will need to partiton the query @@ -269,15 +296,18 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t QoQExec.addPartition( partitionKey, intersection ); } ); + // Make stream parallel if we have a lot of partitions var partitionStream = QoQExec.getPartitions().values().stream(); if ( QoQExec.getPartitions().size() > 50 ) { partitionStream = partitionStream.parallel(); } + // Filter out partitions that don't match the having clause if ( having != null ) { partitionStream = partitionStream.filter( partition -> ( Boolean ) having.evaluateAggregate( QoQExec, partition ) ); } + // Build up the final result set partitionStream.forEach( partition -> { Object[] values = new Object[ resultColumns.size() ]; int colPos = 0; @@ -360,6 +390,10 @@ private static Query getSourceQuery( IBoxContext context, QoQSelectStatementExec throw new DatabaseException( "Unknown table type [" + table.getClass().getName() + "]" ); } + /** + * Represent a result column with a runtime type + * TODO: We may not need this since the expression can tell us the type directly + */ public record TypedResultColumn( QueryColumnType type, SQLResultColumn resultColumn ) { public static TypedResultColumn of( QueryColumnType type, SQLResultColumn resultColumn ) { @@ -367,6 +401,7 @@ public static TypedResultColumn of( QueryColumnType type, SQLResultColumn result } } + // Represent the name and order of an order by statement. This is calculated at runtime since the actual column names may be based on a * public record NameAndDirection( Key name, boolean ascending ) { public static NameAndDirection of( Key name, boolean ascending ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index 51863e8ca..7efaa5c0a 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -103,11 +103,29 @@ public class QoQFunctionService { private QoQFunctionService() { } + /** + * Register a scalar function via a BiFunction + * + * @param name The name of the function + * @param function The function to execute + * @param functionDef The function definition + * @param returnType The return type of the function + * @param requiredParams The number of required parameters + */ public static void register( Key name, java.util.function.BiFunction, List, Object> function, IQoQFunctionDef functionDef, QueryColumnType returnType, int requiredParams ) { functions.put( name, QoQFunction.of( function, functionDef, returnType, requiredParams ) ); } + /** + * Register a custom function based on a UDF or closure + * + * @param name The name of the function + * @param function The function to execute + * @param returnType The return type of the function + * @param requiredParams The number of required parameters + * @param context The context to execute the function in + */ public static void registerCustom( Key name, ortus.boxlang.runtime.types.Function function, QueryColumnType returnType, int requiredParams, IBoxContext context ) { functions.put( name, QoQFunction.of( @@ -119,6 +137,15 @@ public static void registerCustom( Key name, ortus.boxlang.runtime.types.Functio ) ); } + /** + * Register an aggregate function via a BiFunction + * + * @param name The name of the function + * @param function The function to execute + * @param functionDef The function definition + * @param returnType The return type of the function + * @param requiredParams The number of required parameters + */ public static void registerAggregate( Key name, java.util.function.BiFunction, List, Object> function, IQoQFunctionDef functionDef, QueryColumnType returnType, @@ -126,18 +153,40 @@ public static void registerAggregate( Key name, java.util.function.BiFunction
  • , List, Object> callable, java.util.function.BiFunction, List, Object> aggregateCallable, @@ -152,30 +204,78 @@ public record QoQFunction( QueryColumnType returnType, int requiredParams ) { + /** + * static factory method to create scalar function + * + * @param callable The function to execute + * @param functionDef The function definition + * @param returnType The return type of the function + * @param requiredParams The number of required parameters + * + * @return The function + */ static QoQFunction of( java.util.function.BiFunction, List, Object> callable, IQoQFunctionDef functionDef, QueryColumnType returnType, int requiredParams ) { return new QoQFunction( callable, null, functionDef, returnType, requiredParams ); } + /** + * static factory method to create aggregate function + * + * @param callable The function to execute + * @param functionDef The function definition + * @param returnType The return type of the function + * @param requiredParams The number of required parameters + * + */ static QoQFunction ofAggregate( java.util.function.BiFunction, List, Object> callable, IQoQFunctionDef functionDef, QueryColumnType returnType, int requiredParams ) { return new QoQFunction( null, callable, functionDef, returnType, requiredParams ); } + /** + * Invoke the scalar function + * + * @param arguments evaluated args + * @param expressions expressions + * + * @return the result of the function + */ public Object invoke( List arguments, List expressions ) { return callable.apply( arguments, expressions ); } + /** + * Invoke the aggregate function + * + * @param arguments evaluated args + * @param expressions expressions + * + * @return the result of the function + */ public Object invokeAggregate( List arguments, List expressions ) { return aggregateCallable.apply( arguments, expressions ); } + /** + * Check if the function is an aggregate function + * + * @return true if the function is an aggregate function + */ public boolean isAggregate() { return aggregateCallable != null; } + /** + * Get the return type of the function + * + * @param QoQExec The QoQ Execution state + * @param expressions The expressions + * + * @return the return type of the function + */ public QueryColumnType returnType( QoQSelectExecution QoQExec, List expressions ) { if ( functionDef != null ) { return functionDef.getReturnType( QoQExec, expressions ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java index 3a06d249c..0cf925e38 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java @@ -78,6 +78,16 @@ public static Stream createIntersectionStream( QoQSelectExecution QoQExec return theStream; } + /** + * Handle CROSS and INNER JOINs + * + * @param theStream The current stream of intersections + * @param joinTable The table to join + * @param joinOn The ON clause + * @param QoQExec The current QoQ execution + * + * @return The new stream of intersections + */ private static Stream handleCrossOrInnerJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { theStream = theStream.flatMap( i -> IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { int[] newIntersection = Arrays.copyOf( i, i.length + 1 ); @@ -90,6 +100,16 @@ private static Stream handleCrossOrInnerJoin( Stream theStream, Qu return theStream; } + /** + * Handle LEFT JOINs + * + * @param theStream The current stream of intersections + * @param joinTable The table to join + * @param joinOn The ON clause + * @param QoQExec The current QoQ execution + * + * @return The new stream of intersections + */ private static Stream handleLeftJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { return theStream.flatMap( i -> { Stream newStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> { @@ -107,6 +127,16 @@ private static Stream handleLeftJoin( Stream theStream, Query join } ); } + /** + * Handle RIGHT JOINs + * + * @param theStream The current stream of intersections + * @param joinTable The table to join + * @param joinOn The ON clause + * @param QoQExec The current QoQ execution + * + * @return The new stream of intersections + */ private static Stream handleRightJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); @@ -129,6 +159,16 @@ private static Stream handleRightJoin( Stream theStream, Query joi } ); } + /** + * Handle FULL OUTER JOINs + * + * @param theStream The current stream of intersections + * @param joinTable The table to join + * @param joinOn The ON clause + * @param QoQExec The current QoQ execution + * + * @return The new stream of intersections + */ private static Stream handleFullOuterJoin( Stream theStream, Query joinTable, SQLExpression joinOn, QoQSelectExecution QoQExec ) { List leftRows = theStream.collect( Collectors.toList() ); // Collect the left rows to avoid reusing the stream Stream rightStream = IntStream.rangeClosed( 1, joinTable.size() ).mapToObj( j -> new int[] { j } ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java index 555d13181..70117dc99 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQScalarFunctionDef.java @@ -23,6 +23,9 @@ */ public abstract class QoQScalarFunctionDef implements IQoQFunctionDef, java.util.function.BiFunction, List, Object> { + /** + * Is this function an aggregate function + */ public boolean isAggregate() { return false; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index 96ffaba86..59935c394 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -64,34 +64,77 @@ private QoQSelectExecution( SQLSelect select, Map tableLookup ) this.tableLookup = tableLookup; } + /** + * Factory method + * + * @param select select AST node + * @param tableLookup table lookup + * + * @return new QoQSelectExecution instance + */ public static QoQSelectExecution of( SQLSelect select, Map tableLookup ) { return new QoQSelectExecution( select, tableLookup ); } + /** + * Get the select node + */ public SQLSelect getSelect() { return select; } + /** + * Get the table lookup + * + * @return table lookup + */ public Map getTableLookup() { return tableLookup; } + /** + * Get the result columns + * + * @return result columns + */ public Map getResultColumns() { return resultColumns; } + /** + * Set the result columns + * + * @param resultColumns result columns + */ public void setResultColumns( Map resultColumns ) { this.resultColumns = resultColumns; } + /** + * Get the select statement execution + * + * @return select statement execution + */ public QoQSelectStatementExecution getSelectStatementExecution() { return selectStatementExecution; } + /** + * Set the select statement execution + * + * @param selectStatementExecution select statement execution + */ public void setQoQSelectStatementExecution( QoQSelectStatementExecution selectStatementExecution ) { this.selectStatementExecution = selectStatementExecution; } + /** + * Calculate the result columns + * + * @param firstSelect whether this is the first select + * + * @return result columns + */ public Map calculateResultColumns( boolean firstSelect ) { Map resultColumns = new LinkedHashMap(); for ( SQLResultColumn resultColumn : getSelect().getResultColumns() ) { @@ -139,6 +182,9 @@ public Map calculateResultColumns( boolean firstSelect ) return resultColumns; } + /** + * Calculate the order by columns + */ public void calculateOrderBys() { var QoQStmtExec = selectStatementExecution; SQLSelectStatement selectStatement = QoQStmtExec.selectStatement; @@ -202,14 +248,32 @@ public Query getIndepententSubQuery( SQLSelectStatement subquery ) { ); } + /** + * Add a partition + * + * @param partitionName Name of the partition + * @param partition The partition + */ public void addPartition( String partitionName, int[] partition ) { partitions.computeIfAbsent( partitionName, p -> new ArrayList() ).add( partition ); } + /** + * Get a partition + * + * @param partitionName Name of the partition + * + * @return The partition + */ public List getPartition( String partitionName ) { return partitions.get( partitionName ); } + /** + * Get all partitions + * + * @return All partitions + */ public Map> getPartitions() { return partitions; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java index af2da9400..2c9406d23 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java @@ -55,58 +55,132 @@ private QoQSelectStatementExecution( SQLSelectStatement selectStatement, List params, QoQStatement JDBCStatement ) { return new QoQSelectStatementExecution( selectStatement, params, JDBCStatement ); } + /** + * Get the select statement node + * + * @return The select statement node + */ public SQLSelectStatement getSelectStatement() { return selectStatement; } + /** + * Get the parameters to the query + * + * @return The parameters to the query + */ public List getParams() { return params; } + /** + * Get the result column names + * + * @return The result column names + */ public Set getResultColumnName() { return resultColumnNames; } + /** + * Set the result column names + * + * @param resultColumnNames The result column names + */ public void setResultColumnNames( Set resultColumnNames ) { this.resultColumnNames = resultColumnNames; } + /** + * Get the order by columns + * + * @return The order by columns + */ public List getOrderByColumns() { return orderByColumns; } + /** + * Set the order by columns + * + * @param orderByColumns The order by columns + */ public void setOrderByColumns( List orderByColumns ) { this.orderByColumns = orderByColumns; } + /** + * Get the selects + * + * @return The selects + */ public List getSelects() { return selects; } + /** + * get the additional columns, which are only being selected for the order by + * + * @return The additional columns + */ public Set getAdditionalColumns() { return additionalColumns; } + /** + * Set the additional columns + * + * @param additionalColumns The additional columns + */ public void setAdditionalColumns( Set additionalColumns ) { this.additionalColumns = additionalColumns; } + /** + * Add a select + * + * @param select The select + * + * @return The QoQSelectStatementExecution + */ public QoQSelectStatementExecution addSelect( QoQSelectExecution select ) { select.setQoQSelectStatementExecution( this ); selects.add( select ); return this; } + /** + * Create a new QoQSelectExecution + * + * @param select The select node + * @param tableLookup The table lookup + * + * @return The new QoQSelectExecution + */ public QoQSelectExecution newQoQSelectExecution( SQLSelect select, Map tableLookup ) { var QoQExec = QoQSelectExecution.of( select, tableLookup ); addSelect( QoQExec ); return QoQExec; } + /** + * Get the JDBC statement + * + * @return The JDBC statement + */ public QoQStatement getJDBCStatement() { return JDBCStatement; } diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 8f2c68c1d..1f1dfac8a 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -359,4 +359,138 @@ order by count(1) desc context ); } + @Test + public void testStandardCase() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select name, + case + when name = 'brad' then 'me' + when name = 'luis' then 'boss' + when name = 'jacob' then 'Mr. Kansas City' + else 'other' + end as title + from qryEmployees + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + + @Test + public void testStandardCaseNoElse() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select name, + case + when name = 'brad' then 'me' + when name = 'luis' then 'boss' + when name = 'jacob' then 'Mr. Kansas City' + end as title + from qryEmployees + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + + @Test + public void testInputCase() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select name, + case name + when 'brad' then 'me' + when 'luis' then 'boss' + when 'jacob' then 'Mr. Kansas City' + else 'other' + end as title + from qryEmployees + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + + @Test + public void testInputCaseNoElse() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select name, + case name + when 'brad' then 'me' + when 'luis' then 'boss' + when 'jacob' then 'Mr. Kansas City' + end as title + from qryEmployees + ", + [], + { dbType : "query" } + ); + println( q ) + + """, + context ); + } + } From d0050322410bc223057e49f42969b28fb0b2455c Mon Sep 17 00:00:00 2001 From: Michael Born Date: Wed, 18 Dec 2024 05:39:21 -0500 Subject: [PATCH 027/161] Javadoc cleanup --- .../compiler/ast/visitor/CFTranspilerVisitor.java | 3 +-- .../boxlang/compiler/javaboxpiler/Transpiler.java | 2 -- .../application/BaseApplicationListener.java | 6 +++--- .../ortus/boxlang/runtime/async/BoxFuture.java | 12 +++++------- .../cache/providers/AbstractCacheProvider.java | 4 ++-- .../runtime/config/segments/DatasourceConfig.java | 2 +- .../runtime/config/util/PropertyHelper.java | 2 +- .../ortus/boxlang/runtime/dynamic/Attempt.java | 3 +-- .../boxlang/runtime/interop/MethodRecord.java | 4 ++-- .../boxlang/runtime/jdbc/ChildTransaction.java | 14 +++++++------- .../ortus/boxlang/runtime/jdbc/PendingQuery.java | 8 ++++---- .../ortus/boxlang/runtime/jdbc/QueryParameter.java | 2 +- .../runtime/jdbc/drivers/GenericJDBCDriver.java | 4 ++-- .../java/ortus/boxlang/runtime/types/Query.java | 2 -- .../boxlang/runtime/types/StructMapWrapper.java | 2 +- .../types/unmodifiable/UnmodifiableQuery.java | 2 -- .../boxlang/runtime/types/util/StringUtil.java | 4 +--- 17 files changed, 32 insertions(+), 44 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java b/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java index 33ba3586e..e16eb416a 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java @@ -877,7 +877,7 @@ public BoxNode visit( BoxBinaryOperation node ) { * Rewrite !foo eq bar * as !(foo eq bar) * These operators should be higher precedence than the not operator - * EQ, NEQ, LT, LTE, GT, GTE, ==, !=, >, >=, <, <= + * EQ, NEQ, LT, LTE, GT, GTE, ==, !=, >, >=, <, <= */ public BoxNode visit( BoxComparisonOperation node ) { BoxExpression left = node.getLeft(); @@ -896,7 +896,6 @@ public BoxNode visit( BoxComparisonOperation node ) { * Rewrite !foo eq bar * as !(foo eq bar) * These operators should be higher precedence than the not operator - * & */ public BoxNode visit( BoxStringConcat node ) { List values = node.getValues(); diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/Transpiler.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/Transpiler.java index 8258cb01a..de5bb2a88 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/Transpiler.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/Transpiler.java @@ -185,8 +185,6 @@ public int incrementAndGetForInCounter() { /** * Increment and return the lambda counter - * - * @return the incremented value */ public void pushContextName( String name ) { if ( name == null ) { diff --git a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java index 701c9780a..f63c141ca 100644 --- a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java +++ b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java @@ -467,9 +467,9 @@ public InterceptorPool getInterceptorPool() { /** * Helper to Announce an event with the provided {@link IStruct} of data and the app context * - * @param state The state to announce - * @param data The data to announce - * @param context The application context + * @param state The state to announce + * @param data The data to announce + * @param appContext The application context */ public void announce( BoxEvent state, IStruct data, IBoxContext appContext ) { announce( state.key(), data, appContext ); diff --git a/src/main/java/ortus/boxlang/runtime/async/BoxFuture.java b/src/main/java/ortus/boxlang/runtime/async/BoxFuture.java index 78186d472..73c2e72b9 100644 --- a/src/main/java/ortus/boxlang/runtime/async/BoxFuture.java +++ b/src/main/java/ortus/boxlang/runtime/async/BoxFuture.java @@ -449,9 +449,7 @@ public static BoxFuture ofValue( Object value ) { /** * Creates a new BoxFuture from a CompletableFuture * - * @param - * - * @param future The CompletableFuture to wrap + * @param value The CompletableFuture to wrap * * @return A future that is already completed with the given value */ @@ -506,7 +504,7 @@ public static BoxFuture ofFunction( IBoxContext context, ortus.boxlang.runtim * // Array * allApply( items, ( item ) => item.getMemento() ) * // Struct: The result object is a struct of `key` and `value` - * allApply( data, ( item ) => item.key & item.value.toString() ) + * allApply( data, ( item ) => item.key & item.value.toString() ) * * * @param context The context of the current execution @@ -535,7 +533,7 @@ public static Object allApply( * // Array * allApply( items, ( item ) => item.getMemento() ) * // Struct: The result object is a struct of `key` and `value` - * allApply( data, ( item ) => item.key & item.value.toString() ) + * allApply( data, ( item ) => item.key & item.value.toString() ) * * * @param context The context of the current execution @@ -565,7 +563,7 @@ public static Object allApply( * // Array * allApply( items, ( item ) => item.getMemento() ) * // Struct: The result object is a struct of `key` and `value` - * allApply( data, ( item ) => item.key & item.value.toString() ) + * allApply( data, ( item ) => item.key & item.value.toString() ) * * * @param context The context of the current execution @@ -596,7 +594,7 @@ public static Object allApply( * // Array * allApply( items, ( item ) => item.getMemento() ) * // Struct: The result object is a struct of `key` and `value` - * allApply( data, ( item ) => item.key & item.value.toString() ) + * allApply( data, ( item ) => item.key & item.value.toString() ) * * * @param context The context of the current execution diff --git a/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java b/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java index 735e54005..7f35dcb14 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java +++ b/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java @@ -288,9 +288,9 @@ public void announce( BoxEvent state, IStruct data ) { } /** - * Converts the timeout to a duration. + * Converts the seconds value to a duration. * - * @param seconds The seconds to convert. This can be a duration, number or string representation of a number + * @param timeout The seconds to convert. This can be a duration, number or string representation of a number * * @return The duration of seconds according to the seconds passed */ diff --git a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java index 4a688a2b7..f81b179eb 100644 --- a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java +++ b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java @@ -656,7 +656,7 @@ public String addCustomParams( String target, String URIDelimiter, String delimi /** * This method is used to incorporate custom parameters into the target connection string - * Using default delimiters of ? and & + * Using default delimiters of ? and & * * @param target The target connection string * diff --git a/src/main/java/ortus/boxlang/runtime/config/util/PropertyHelper.java b/src/main/java/ortus/boxlang/runtime/config/util/PropertyHelper.java index 9e086b5c8..58e6eb77f 100644 --- a/src/main/java/ortus/boxlang/runtime/config/util/PropertyHelper.java +++ b/src/main/java/ortus/boxlang/runtime/config/util/PropertyHelper.java @@ -98,7 +98,7 @@ public static void processStringOrArrayToList( IStruct config, Key key, List allowedValues ) { if ( config.containsKey( key ) ) { diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/Attempt.java b/src/main/java/ortus/boxlang/runtime/dynamic/Attempt.java index eaa533498..f326f9715 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/Attempt.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/Attempt.java @@ -27,7 +27,6 @@ import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.scopes.Key; -import ortus.boxlang.runtime.types.Function; import ortus.boxlang.runtime.types.Struct; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.types.exceptions.CustomException; @@ -597,7 +596,7 @@ public Attempt map( java.util.function.Function m * an empty {@code Attempt}. * *

    - * This method is similar to {@link #map(Function)}, but the mapping + * This method is similar to {@link #map( java.util.function.Function mapper )}, but the mapping * function is one whose result is already an {@code Attempt}, and if * invoked, {@code flatMap} does not wrap it within an additional * {@code Attempt}. diff --git a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java index 7f5493b73..1b8fd359c 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java +++ b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java @@ -25,7 +25,7 @@ * This record is the one that is cached in the {@link DynamicObject} map. * * @param methodName The name of the method - * @param defaultMethod The method representation + * @param method The method representation * @param methodHandle The method handle to use for invocation * @param isStatic Whether the method is static or not * @param argumentCount The number of arguments the method takes @@ -36,5 +36,5 @@ public record MethodRecord( MethodHandle methodHandle, boolean isStatic, int argumentCount ) { - // A beautiful java record of our method handle + // A beautiful java record of our method handle } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ChildTransaction.java b/src/main/java/ortus/boxlang/runtime/jdbc/ChildTransaction.java index 27aad2529..0b850eb2b 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ChildTransaction.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ChildTransaction.java @@ -1,11 +1,11 @@ package ortus.boxlang.runtime.jdbc; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import ortus.boxlang.runtime.scopes.Key; /** @@ -13,10 +13,10 @@ *

    * Utilizes savepoints to manage the nested transaction state: *

      - *
    • On transaction begin, a savepoint is created.
      • - *
    • On transaction commit, a COMMIT savepoint is created.
      • - *
    • On transaction rollback, the transaction is rolled back to the child transaction's BEGIN savepoint. (Unless a savepoint name is passed, in which case the transaction is rolled back to that savepoint only.)
      • - *
    • On transaction end, an END savepoint is created.
      • + *
    • On transaction begin, a <code>BEGIN<code> savepoint is created.
      • + *
    • On transaction commit, a <code>COMMIT<code> savepoint is created.
      • + *
    • On transaction rollback, the transaction is rolled back to the child transaction's <code>BEGIN<code> savepoint. (Unless a savepoint name is passed, in which case the transaction is rolled back to that savepoint only.)
      • + *
    • On transaction end, an <code>END<code> savepoint is created.
      • *
    *

    * The savepoint names are prefixed with a unique identifier to ensure that they don't collide with savepoints created in the parent transaction. diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index 409c8b674..c942c1cf4 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -49,7 +49,7 @@ /** * This class represents a query and any parameters/bindings before being executed. - * After calling {@link #execute()}, it returns an {@link ExecutedQuery} with a reference to this object. + * After calling {@link #execute(ConnectionManager,IBoxContext)}, it returns an {@link ExecutedQuery} with a reference to this object. */ public class PendingQuery { @@ -123,9 +123,9 @@ public class PendingQuery { /** * Creates a new PendingQuery instance from a SQL string, a list of parameters, and the original SQL string. * - * @param sql The SQL string to execute - * @param parameters A list of {@link QueryParameter} to use as bindings. - * @param originalSql The original sql string. This will include named parameters if the `PendingQuery` was constructed using an {@link IStruct}. + * @param sql The SQL string to execute + * @param bindings An array or struct of {@link QueryParameter} to use as bindings. + * @param queryOptions QueryOptions object denoting the options for this query. */ public PendingQuery( @Nonnull String sql, Object bindings, QueryOptions queryOptions ) { logger.debug( "Building new PendingQuery from SQL: [{}] and options: [{}]", sql, queryOptions.toStruct() ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index dbd7517cb..84eb4d97e 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -167,7 +167,7 @@ public Integer getScaleOrLength() { * Returns whether this parameter has a null value override. *

    * For example: - * `` + * <bx:queryparam null="true" /> */ public boolean isNull() { return this.isNullParam; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/drivers/GenericJDBCDriver.java b/src/main/java/ortus/boxlang/runtime/jdbc/drivers/GenericJDBCDriver.java index 803e333d4..da0dd135c 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/drivers/GenericJDBCDriver.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/drivers/GenericJDBCDriver.java @@ -60,13 +60,13 @@ public class GenericJDBCDriver implements IJDBCDriver { /** * The default delimiter for the addition of custom paramters to the connection URL. * This delimits the start of the custom parameters. - * Example: jdbc:mysql://localhost:3306/mydb?param1=value1¶m2=value2 + * Example: jdbc:mysql://localhost:3306/mydb?param1=value1&param2=value2 */ protected String defaultURIDelimiter = "?"; /** * The default delimiter for the custom parameters attached to the connection URL - * Example: jdbc:mysql://localhost:3306/mydb?param1=value1¶m2=value2 + * Example: jdbc:mysql://localhost:3306/mydb?param1=value1&param2=value2 * This delimits each custom parameter. */ protected String defaultDelimiter = "&"; diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index e368b6f14..3a218de7c 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -110,8 +110,6 @@ public Query() { * Create a new query and populate it from the given JDBC ResultSet. * * @param resultSet JDBC result set. - * - * @return Query object */ public static Query fromResultSet( ResultSet resultSet ) { Query query = new Query(); diff --git a/src/main/java/ortus/boxlang/runtime/types/StructMapWrapper.java b/src/main/java/ortus/boxlang/runtime/types/StructMapWrapper.java index 74a11f740..1e17d42e8 100644 --- a/src/main/java/ortus/boxlang/runtime/types/StructMapWrapper.java +++ b/src/main/java/ortus/boxlang/runtime/types/StructMapWrapper.java @@ -103,7 +103,7 @@ public class StructMapWrapper implements IStruct, IListenable, Serializable { /** * Constructor * - * @param type The type of struct to create: DEFAULT, LINKED, SORTED + * @param map The map to wrap * * @throws BoxRuntimeException If an invalid type is specified: DEFAULT, LINKED, SORTED */ diff --git a/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java b/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java index 8cffc4b26..1c91450cd 100644 --- a/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java +++ b/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java @@ -64,8 +64,6 @@ public UnmodifiableQuery() { * Create an Unmodifiable query from a Modifiable query * * @param query The Modifiable query to convert - * - * @return The Unmodifiable query */ public UnmodifiableQuery( Query query ) { this(); diff --git a/src/main/java/ortus/boxlang/runtime/types/util/StringUtil.java b/src/main/java/ortus/boxlang/runtime/types/util/StringUtil.java index de12a8fa2..eb339e1a9 100644 --- a/src/main/java/ortus/boxlang/runtime/types/util/StringUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/util/StringUtil.java @@ -87,9 +87,7 @@ public class StringUtil { /** * Slugify a string for URL Safety * - * @param str Target to slugify - * @param maxLength The maximum number of characters for the slug - * @param allow a regex safe list of additional characters to allow + * @param str Target to slugify */ public static String slugify( String str ) { return slugify( str, 0, "" ); From 391b6f9d58d6bf8cd8f70d683fd9c2a4ce793521 Mon Sep 17 00:00:00 2001 From: michaelborn Date: Wed, 18 Dec 2024 10:40:11 +0000 Subject: [PATCH 028/161] Apply cfformat changes --- src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java index 1b8fd359c..4ac2985e0 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java +++ b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java @@ -36,5 +36,5 @@ public record MethodRecord( MethodHandle methodHandle, boolean isStatic, int argumentCount ) { - // A beautiful java record of our method handle + // A beautiful java record of our method handle } From f17b16c3974ae449ee4c768ef8792a07f79010cd Mon Sep 17 00:00:00 2001 From: Michael Born Date: Wed, 18 Dec 2024 06:46:17 -0500 Subject: [PATCH 029/161] Apply spotless java formatting --- src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java index 1b8fd359c..4ac2985e0 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java +++ b/src/main/java/ortus/boxlang/runtime/interop/MethodRecord.java @@ -36,5 +36,5 @@ public record MethodRecord( MethodHandle methodHandle, boolean isStatic, int argumentCount ) { - // A beautiful java record of our method handle + // A beautiful java record of our method handle } From 9763d533531742c5614e5bda946f098d6d4d1290 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 11:51:44 -0600 Subject: [PATCH 030/161] BL-865 --- .../boxlang/compiler/ast/visitor/CFTranspilerVisitor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java b/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java index e16eb416a..e9825f839 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/ast/visitor/CFTranspilerVisitor.java @@ -223,8 +223,10 @@ public BoxNode visit( BoxClass node ) { mergeDocsIntoAnnotations( annotations, node.getDocumentation() ); // Disable Accessors by default in CFML, unless there is a parent class, in which case don't add so we can inherit + // Also, if this is a persistent ORM entity, then leave as-is because accessors needs to be enabled anyway if ( annotations.stream().noneMatch( a -> a.getKey().getValue().equalsIgnoreCase( "accessors" ) ) - && annotations.stream().noneMatch( a -> a.getKey().getValue().equalsIgnoreCase( "extends" ) ) ) { + && annotations.stream().noneMatch( a -> a.getKey().getValue().equalsIgnoreCase( "extends" ) ) + && annotations.stream().noneMatch( a -> a.getKey().getValue().equalsIgnoreCase( "persistent" ) ) ) { // @output true annotations.add( new BoxAnnotation( From 1d5acc72d082dfd79f29240481bfbdf9d1c99369 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 13:00:31 -0600 Subject: [PATCH 031/161] BL-866 --- .../types/exceptions/ExceptionUtil.java | 22 ++++++++++-------- .../java/TestCases/phase3/ExceptionTest.java | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java index 3a1648844..b6f5b3d22 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java @@ -64,16 +64,20 @@ public static Boolean exceptionIsOfType( IBoxContext context, Throwable e, Strin if ( type.equalsIgnoreCase( "any" ) ) { return true; } - // BoxLangExceptions check the type - if ( e instanceof BoxLangException ble ) { - // Either direct match to type, or "foo.bar" matches "foo.bar.baz - if ( ble.type.equalsIgnoreCase( type ) || ble.type.toLowerCase().startsWith( type + "." ) ) - return true; - } + // Check the exception and all of its causes + while ( e != null ) { + // BoxLangExceptions check the type + if ( e instanceof BoxLangException ble ) { + // Either direct match to type, or "foo.bar" matches "foo.bar.baz + if ( ble.type.equalsIgnoreCase( type ) || ble.type.toLowerCase().startsWith( type + "." ) ) + return true; + } - // Native exceptions just check the class hierarchy - if ( InstanceOf.invoke( context, e, type ) ) { - return true; + // Native exceptions just check the class hierarchy + if ( InstanceOf.invoke( context, e, type ) ) { + return true; + } + e = e.getCause(); } return false; } diff --git a/src/test/java/TestCases/phase3/ExceptionTest.java b/src/test/java/TestCases/phase3/ExceptionTest.java index 320b17f9a..10d7587e3 100644 --- a/src/test/java/TestCases/phase3/ExceptionTest.java +++ b/src/test/java/TestCases/phase3/ExceptionTest.java @@ -58,6 +58,7 @@ public void setupEach() { @Test public void testBoxMeta() { + // TODO: I'm not sure what the point of this test is. It seems I got distracted before adding the actual assertions? instance.executeStatement( """ include "src/test/java/TestCases/phase3/ExceptionThrower.cfs"; @@ -67,4 +68,26 @@ public void testBoxMeta() { } + @Test + public void testCatchInnerType() { + + instance.executeStatement( + """ + try{ + try { + throw( type="inner", message="inner" ) + } catch( e ) { + throw ( type="outer", message="outer", object=e ) + } + } catch( inner e ) { + result = e + } + """, context ); + + Throwable t = ( Throwable ) variables.get( result ); + assertThat( t.getMessage() ).isEqualTo( "outer" ); + assertThat( t.getCause().getMessage() ).isEqualTo( "inner" ); + + } + } From 8425cffd4f44c7c4d652205df7db97ae9c710f93 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 15:10:54 -0600 Subject: [PATCH 032/161] BL-849 --- .../compiler/asmboxpiler/AsmTranspiler.java | 4 ++- .../transformer/BoxClassTransformer.java | 7 ++-- .../runtime/runnables/BoxClassSupport.java | 12 ++++--- .../ortus/boxlang/runtime/types/Property.java | 9 +++-- .../boxlang/runtime/types/meta/ClassMeta.java | 19 +++++----- src/test/java/TestCases/phase3/ClassTest.java | 35 +++++++++++++++++++ .../PropertiesNotInheritedInMetadata.bx | 5 +++ .../PropertiesNotInheritedInMetadataParent.bx | 5 +++ 8 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadata.bx create mode 100644 src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadataParent.bx diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java index 5ced452e5..f5d6ce106 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java @@ -790,12 +790,14 @@ public List> transformProperties( Type declaringType, Lis sourceType.toUpperCase(), Type.getDescriptor( BoxSourceType.class ) ) ); + javaExpr.add( new LdcInsnNode( declaringType ) ); + javaExpr.add( new MethodInsnNode( Opcodes.INVOKESPECIAL, Type.getInternalName( Property.class ), "", Type.getMethodDescriptor( Type.VOID_TYPE, Type.getType( Key.class ), Type.getType( String.class ), Type.getType( Object.class ), Type.getType( DefaultExpression.class ), Type.getType( IStruct.class ), Type.getType( IStruct.class ), - Type.getType( BoxSourceType.class ) ), + Type.getType( BoxSourceType.class ), Type.getType( Class.class ) ), false ) ); members.add( jNameKey ); diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/BoxClassTransformer.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/BoxClassTransformer.java index 1d1dd56bf..447aea32f 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/BoxClassTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/BoxClassTransformer.java @@ -580,7 +580,7 @@ public Node transform( BoxNode node, TransformerContext context ) throws Illegal result.getResult().orElseThrow().getType( 0 ).getFieldByName( "documentation" ).orElseThrow().getVariable( 0 ) .setInitializer( documentationStruct ); - List propertyStructs = transformProperties( boxClass.getProperties(), sourceType ); + List propertyStructs = transformProperties( boxClass.getProperties(), sourceType, className ); result.getResult().orElseThrow().getType( 0 ).getFieldByName( "properties" ).orElseThrow().getVariable( 0 ) .setInitializer( propertyStructs.get( 0 ) ); result.getResult().orElseThrow().getType( 0 ).getFieldByName( "getterLookup" ).orElseThrow().getVariable( 0 ) @@ -691,7 +691,7 @@ private String generateCompileTimeMethodNames( BoxClass boxClass ) { * * @return A list of the [ properties, getters, setters] */ - private List transformProperties( List properties, String sourceType ) { + private List transformProperties( List properties, String sourceType, String className ) { List members = new ArrayList<>(); List getterLookup = new ArrayList<>(); List setterLookup = new ArrayList<>(); @@ -764,8 +764,9 @@ private List transformProperties( List properties, Stri values.put( "annotations", annotationStruct.toString() ); values.put( "documentation", documentationStruct.toString() ); values.put( "sourceType", sourceType ); + values.put( "className", className ); String template = """ - new Property( ${name}, "${type}", ${defaultValue}, ${defaultExpression}, ${annotations} ,${documentation}, BoxSourceType.${sourceType} ) + new Property( ${name}, "${type}", ${defaultValue}, ${defaultExpression}, ${annotations} ,${documentation}, BoxSourceType.${sourceType}, ${className}.class ) """; Expression javaExpr = parseExpression( template, values ); // logger.trace( "{} -> {}", prop.getSourceText(), javaExpr ); diff --git a/src/main/java/ortus/boxlang/runtime/runnables/BoxClassSupport.java b/src/main/java/ortus/boxlang/runtime/runnables/BoxClassSupport.java index 88e4699d7..044e80c52 100644 --- a/src/main/java/ortus/boxlang/runtime/runnables/BoxClassSupport.java +++ b/src/main/java/ortus/boxlang/runtime/runnables/BoxClassSupport.java @@ -70,9 +70,9 @@ public static void pseudoConstructor( IClassRunnable thisClass, IBoxContext cont } /** - * Call the pseudo constructor + * I handle creating default values for all properties defined * - * @param thisClass The class to call the pseudo constructor on + * @param thisClass The class to create default properties for * @param context The context to use */ public static void defaultProperties( IClassRunnable thisClass, IBoxContext context ) { @@ -495,8 +495,12 @@ public static IStruct getMetaData( IClassRunnable thisClass ) { var properties = new Array(); // loop over properties list and add struct for each property for ( var entry : thisClass.getProperties().entrySet() ) { - var property = entry.getValue(); - var propertyStruct = new Struct( IStruct.TYPES.LINKED ); + var property = entry.getValue(); + // Only include properties declared here, not in a parent/extends + if ( property.declaringClass() != thisClass.getClass() ) { + continue; + } + var propertyStruct = new Struct( IStruct.TYPES.LINKED ); propertyStruct.put( "name", property.name().getName() ); propertyStruct.put( "type", property.type() ); if ( property.hasDefaultValue() ) { diff --git a/src/main/java/ortus/boxlang/runtime/types/Property.java b/src/main/java/ortus/boxlang/runtime/types/Property.java index c0d660a27..25f8c4afb 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Property.java +++ b/src/main/java/ortus/boxlang/runtime/types/Property.java @@ -35,14 +35,16 @@ * @param documentation Documentation for the property * @param generatedGetter The generated getter Function * @param generatedSetter The generated setter Function + * @param sourceType The source type of the property + * @param declaringClass The class that declares this property * */ public record Property( Key name, String type, Object defaultValue, DefaultExpression defaultExpression, IStruct annotations, IStruct documentation, Key getterName, Key setterName, - UDF generatedGetter, UDF generatedSetter, BoxSourceType sourceType ) { + UDF generatedGetter, UDF generatedSetter, BoxSourceType sourceType, Class declaringClass ) { public Property( Key name, String type, Object defaultValue, DefaultExpression defaultExpression, IStruct annotations, IStruct documentation, - BoxSourceType sourceType ) { + BoxSourceType sourceType, Class declaringClass ) { // Pre-calculate the getter and setter names this( name, @@ -56,7 +58,8 @@ public Property( Key name, String type, Object defaultValue, DefaultExpression d new GeneratedGetter( Key.of( "get" + name.getName() ), name, sourceType.equals( BoxSourceType.CFSCRIPT ) || sourceType.equals( BoxSourceType.CFSCRIPT ) ? "any" : type ), new GeneratedSetter( Key.of( "set" + name.getName() ), name, type ), - sourceType + sourceType, + declaringClass ); } diff --git a/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java b/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java index 499eebb6c..36edd7d99 100644 --- a/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java +++ b/src/main/java/ortus/boxlang/runtime/types/meta/ClassMeta.java @@ -70,14 +70,17 @@ public ClassMeta( IClassRunnable target ) { Key._IMPLEMENTS, UnmodifiableArray.fromList( target.getInterfaces().stream().map( iface -> iface.getBoxMeta().getMeta() ).toList() ), Key.functions, UnmodifiableArray.fromList( functions ), Key._HASHCODE, target.hashCode(), - Key.properties, UnmodifiableArray.of( target.getProperties().entrySet().stream().map( entry -> UnmodifiableStruct.of( - Key._NAME, entry.getKey().getName(), - Key.nameAsKey, entry.getKey(), - Key.type, entry.getValue().type(), - Key.defaultValue, entry.getValue().getDefaultValueForMeta(), - Key.annotations, UnmodifiableStruct.fromStruct( entry.getValue().annotations() ), - Key.documentation, UnmodifiableStruct.fromStruct( entry.getValue().documentation() ) - ) ).toArray() ), + Key.properties, + // Only include properties that are declared in the class + UnmodifiableArray.of( target.getProperties().entrySet().stream().filter( p -> p.getValue().declaringClass() == target.getClass() ) + .map( entry -> UnmodifiableStruct.of( + Key._NAME, entry.getKey().getName(), + Key.nameAsKey, entry.getKey(), + Key.type, entry.getValue().type(), + Key.defaultValue, entry.getValue().getDefaultValueForMeta(), + Key.annotations, UnmodifiableStruct.fromStruct( entry.getValue().annotations() ), + Key.documentation, UnmodifiableStruct.fromStruct( entry.getValue().documentation() ) + ) ).toArray() ), Key.type, "Component", Key.fullname, target.bxGetName().getName(), Key.path, target.getRunnablePath().absolutePath().toString() diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 458cbcc7c..5e7d2daf9 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1566,4 +1566,39 @@ public void testClassLocatorInStaticInitializer() { context ); } + @DisplayName( "properties not inherited in metadata" ) + @Test + public void testPropertiesNotInheritedInMetadata() { + instance.executeSource( + """ + c = new src.test.java.TestCases.phase3.PropertiesNotInheritedInMetadata(); + result = getMetaData( c ) + result2 = c.$bx.meta + """, + context ); + + // Legacy metadata + Array childProps = variables.getAsStruct( Key.of( "result" ) ).getAsArray( Key.properties ); + assertThat( childProps ).hasSize(2); + assertThat( ((IStruct)childProps.get(0)).get( Key._NAME ) ).isEqualTo( "childProperty" ); + assertThat( ((IStruct)childProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + Array parentProps = variables.getAsStruct( Key.of( "result" ) ).getAsStruct(Key._EXTENDS).getAsArray( Key.properties ); + assertThat( parentProps ).hasSize(2); + assertThat( ((IStruct)parentProps.get(0)).get( Key._NAME ) ).isEqualTo( "parentProperty" ); + assertThat( ((IStruct)parentProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + // $bx metadata + childProps = variables.getAsStruct( Key.of( Key.of("result2") ) ).getAsArray( Key.properties ); + assertThat( childProps ).hasSize(2); + assertThat( ((IStruct)childProps.get(0)).get( Key._NAME ) ).isEqualTo( "childProperty" ); + assertThat( ((IStruct)childProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + parentProps = variables.getAsStruct( Key.of( Key.of("result2")) ).getAsStruct(Key._EXTENDS).getAsArray( Key.properties ); + assertThat( parentProps ).hasSize(2); + assertThat( ((IStruct)parentProps.get(0)).get( Key._NAME ) ).isEqualTo( "parentProperty" ); + assertThat( ((IStruct)parentProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + } + } diff --git a/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadata.bx b/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadata.bx new file mode 100644 index 000000000..b1f0a016f --- /dev/null +++ b/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadata.bx @@ -0,0 +1,5 @@ +class extends=PropertiesNotInheritedInMetadataParent { + property name=childProperty; + property name=sharedProperty; + +} \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadataParent.bx b/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadataParent.bx new file mode 100644 index 000000000..d3c3569c0 --- /dev/null +++ b/src/test/java/TestCases/phase3/PropertiesNotInheritedInMetadataParent.bx @@ -0,0 +1,5 @@ +class { + property name=parentProperty; + property name=sharedProperty; + +} \ No newline at end of file From 18eab7986f4b36b9a9c737987d2d848660642b54 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 15:11:37 -0600 Subject: [PATCH 033/161] BL-850 --- .../java/TestCases/phase1/CoreLangTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index d1564429e..b6f1e032a 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -4161,4 +4161,42 @@ function doesStuff() { context, BoxSourceType.CFSCRIPT ); assertThat( variables.getAsString( result ) ).isEqualTo( "set this time" ); } + + + @DisplayName( "dump order" ) + @Test + public void testDumpOrder() { + // @formatter:off + instance.executeSource( + """ + import java.time.Duration; + + // Dumps to Console + writedump( + var : getModuleList(), + label : "Module List", + output : "console" + ); + writeDump( + var : [1,2,3,4,5,6,7,8,9,10], + label : "Array", + output : "console" + ); + writeDump( + var : {a:1,b:2,c:3,d:4,e:5,f:6,g:7,h:8,i:9,j:10}, + label : "Struct", + output : "console" + ); + + writeDump( + var : createObject( "java", "java.time.Instant" ).now(), + label : "Instant", + output : "console" + ); + + writeDump( var=Duration.ofHours(2).plusMinutes(30), label="Duration" ); + """, + context ); + // @formatter:on + } } From 76ee1b5f00c433a1d0c44f0490fe14a5316ac5f9 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 15:11:48 -0600 Subject: [PATCH 034/161] Update ClassLocatorInStaticInitializer.bx --- .../java/TestCases/phase3/ClassLocatorInStaticInitializer.bx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx index c9a9d8b48..286b5156a 100644 --- a/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx +++ b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx @@ -1,5 +1,6 @@ class { static { - createObject( "java", "java.io.File" ) + createObject( "java", "java.io.File" ); + final slashdir = createObject( "java", "java.io.File" ).separator; } } \ No newline at end of file From 382755a8c7de9171df6a3c7a51f1e7c2325e7c50 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 15:36:36 -0600 Subject: [PATCH 035/161] add test for relative self instantiation --- src/test/java/TestCases/phase3/ClassTest.java | 12 ++++++++++++ .../java/TestCases/phase3/RelativeInstantiation.bx | 3 +++ .../TestCases/phase3/RelativeSelfInstantiation.bx | 9 +++++++++ .../bifs/global/system/GetComponentListTest.java | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/test/java/TestCases/phase3/RelativeSelfInstantiation.bx diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 5e7d2daf9..6ea487aae 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1343,8 +1343,20 @@ public void testRelativeInstantiation() { """ clazz = new src.test.java.TestCases.phase3.RelativeInstantiation(); result = clazz.findSibling() + result2 = getMetadata( clazz.returnSibling() ) """, context ); assertThat( variables.get( result ) ).isEqualTo( "bar" ); + assertThat( variables.getAsStruct( Key.of("result2") ).getAsString(Key._NAME ) ).isEqualTo("src.test.java.TestCases.phase3.FindMe"); + } + + @Test + public void testRelativeSelfInstantiation() { + instance.executeSource( + """ + clazz = new src.test.java.TestCases.phase3.RelativeSelfInstantiation(); + result2 = getMetadata( clazz.clone() ) + """, context ); + assertThat( variables.getAsStruct( Key.of("result2") ).getAsString(Key._NAME ) ).isEqualTo("src.test.java.TestCases.phase3.RelativeSelfInstantiation"); } @Test diff --git a/src/test/java/TestCases/phase3/RelativeInstantiation.bx b/src/test/java/TestCases/phase3/RelativeInstantiation.bx index 4e7e792d5..4f87a8ebb 100644 --- a/src/test/java/TestCases/phase3/RelativeInstantiation.bx +++ b/src/test/java/TestCases/phase3/RelativeInstantiation.bx @@ -2,4 +2,7 @@ class { function findSibling() { return new FindMe().foo(); } + function returnSibling() { + return new FindMe(); + } } \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/RelativeSelfInstantiation.bx b/src/test/java/TestCases/phase3/RelativeSelfInstantiation.bx new file mode 100644 index 000000000..7d4d1153a --- /dev/null +++ b/src/test/java/TestCases/phase3/RelativeSelfInstantiation.bx @@ -0,0 +1,9 @@ +class { + RelativeSelfInstantiation function init() { + return this; + } + + function clone() { + return new RelativeSelfInstantiation(); + } +} \ No newline at end of file diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetComponentListTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetComponentListTest.java index 9b1906062..94a21831f 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetComponentListTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetComponentListTest.java @@ -62,7 +62,7 @@ public void testGetComponentList() { context ); // @formatter:on - System.out.println( variables.get( result ) ); + // System.out.println( variables.get( result ) ); IStruct functions = ( IStruct ) variables.get( result ); assertThat( functions ).isNotNull(); From 390e96a761d0d8b81ea133e0458b9d6b8336f9f0 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 18 Dec 2024 15:58:58 -0600 Subject: [PATCH 036/161] BL-868 --- .../bifs/global/cache/CacheFilter.java | 6 ++++-- .../providers/AbstractCacheProvider.java | 6 ++++-- .../cache/providers/BoxCacheProvider.java | 12 ++++++----- .../store/ConcurrentSoftReferenceStore.java | 20 +++++++++++-------- .../runtime/cache/store/ConcurrentStore.java | 14 ++++++++----- .../runtime/cache/store/FileSystemStore.java | 11 ++++++---- .../runtime/components/cache/Cache.java | 11 +++++----- src/test/java/TestCases/phase3/ClassTest.java | 1 + 8 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/cache/CacheFilter.java b/src/main/java/ortus/boxlang/runtime/bifs/global/cache/CacheFilter.java index 18107b959..3e16c9079 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/cache/CacheFilter.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/cache/CacheFilter.java @@ -20,6 +20,8 @@ import ortus.boxlang.runtime.cache.filters.RegexFilter; import ortus.boxlang.runtime.cache.filters.WildcardFilter; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.scopes.ArgumentsScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Argument; @@ -72,8 +74,8 @@ public CacheFilter() { * @return The cache instance that was requested by name. */ public ICacheKeyFilter _invoke( IBoxContext context, ArgumentsScope arguments ) { - Boolean useRegex = arguments.getAsBoolean( Key.useRegex ); - String filter = arguments.getAsString( Key.filter ); + Boolean useRegex = BooleanCaster.cast( arguments.get( Key.useRegex ) ); + String filter = StringCaster.cast( arguments.get( Key.filter ) ); // Build the right filter return useRegex ? new RegexFilter( filter ) : new WildcardFilter( filter ); diff --git a/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java b/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java index 7f35dcb14..403b2f6a9 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java +++ b/src/main/java/ortus/boxlang/runtime/cache/providers/AbstractCacheProvider.java @@ -29,7 +29,9 @@ import ortus.boxlang.runtime.cache.util.ICacheStats; import ortus.boxlang.runtime.config.segments.CacheConfig; import ortus.boxlang.runtime.dynamic.Attempt; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; import ortus.boxlang.runtime.dynamic.casters.LongCaster; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.events.BoxEvent; import ortus.boxlang.runtime.events.InterceptorPool; import ortus.boxlang.runtime.scopes.Key; @@ -254,7 +256,7 @@ protected ExecutorRecord getTaskScheduler() { * JVM Threshold checks */ protected boolean memoryThresholdCheck() { - var threshold = this.config.properties.getAsInteger( Key.freeMemoryPercentageThreshold ); + var threshold = IntegerCaster.cast( this.config.properties.get( Key.freeMemoryPercentageThreshold ) ); // Is it enabled or not if ( threshold == 0 ) { @@ -310,7 +312,7 @@ public static Duration toDuration( Object timeout ) { */ protected static IObjectStore buildObjectStore( CacheConfig config ) { // Store Type - Object thisStore = config.properties.getAsString( Key.objectStore ); + Object thisStore = StringCaster.cast( config.properties.get( Key.objectStore ) ); // Is this a store object already? if ( thisStore instanceof IObjectStore castedStore ) { diff --git a/src/main/java/ortus/boxlang/runtime/cache/providers/BoxCacheProvider.java b/src/main/java/ortus/boxlang/runtime/cache/providers/BoxCacheProvider.java index 25ba30ec5..c799c465e 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/providers/BoxCacheProvider.java +++ b/src/main/java/ortus/boxlang/runtime/cache/providers/BoxCacheProvider.java @@ -34,6 +34,8 @@ import ortus.boxlang.runtime.cache.store.IObjectStore; import ortus.boxlang.runtime.config.segments.CacheConfig; import ortus.boxlang.runtime.dynamic.Attempt; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; import ortus.boxlang.runtime.events.BoxEvent; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.services.CacheService; @@ -131,11 +133,11 @@ public synchronized ICacheProvider configure( CacheService cacheService, CacheCo // Enable reporting this.reportingEnabled = true; // Default Max Size - this.maxObjects = config.properties.getAsInteger( Key.maxObjects ); + this.maxObjects = IntegerCaster.cast( config.properties.get( Key.maxObjects ) ); // Store default timeouts - this.defaultTimeout = Duration.ofSeconds( config.properties.getAsInteger( Key.defaultTimeout ).longValue() ); - this.defaultLastAccessTimeout = Duration.ofSeconds( config.properties.getAsInteger( Key.defaultLastAccessTimeout ).longValue() ); - Long frequency = config.properties.getAsInteger( Key.reapFrequency ).longValue(); + this.defaultTimeout = Duration.ofSeconds( IntegerCaster.cast( config.properties.get( Key.defaultTimeout ) ).longValue() ); + this.defaultLastAccessTimeout = Duration.ofSeconds( IntegerCaster.cast( config.properties.get( Key.defaultLastAccessTimeout ) ).longValue() ); + Long frequency = IntegerCaster.cast( config.properties.get( Key.reapFrequency ) ).longValue(); // Create the reaping scheduled task using the CacheService executor this.reapingFuture = this.cacheService.getTaskScheduler() @@ -280,7 +282,7 @@ public synchronized void reap() { } // Last Access Timeout - if ( config.properties.getAsBoolean( Key.useLastAccessTimeouts ) && + if ( BooleanCaster.cast( config.properties.get( Key.useLastAccessTimeouts ) ) && entry.lastAccessTimeout() > 0 && entry.lastAccessed().plusSeconds( entry.lastAccessTimeout() ).isBefore( rightNow ) ) { clear( entry.key().getName() ); diff --git a/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentSoftReferenceStore.java b/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentSoftReferenceStore.java index 932eabd59..2f72fa3f9 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentSoftReferenceStore.java +++ b/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentSoftReferenceStore.java @@ -29,6 +29,8 @@ import ortus.boxlang.runtime.cache.ICacheEntry; import ortus.boxlang.runtime.cache.filters.ICacheKeyFilter; import ortus.boxlang.runtime.cache.providers.ICacheProvider; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; @@ -75,16 +77,17 @@ public ConcurrentSoftReferenceStore() { */ @Override public IObjectStore init( ICacheProvider provider, IStruct config ) { - this.provider = provider; - this.config = config; - this.pool = new ConcurrentHashMap<>( config.getAsInteger( Key.maxObjects ) / 4 ); - this.softRefKeyMap = new ConcurrentHashMap<>( config.getAsInteger( Key.maxObjects ) / 4 ); + this.provider = provider; + this.config = config; + int maxObjects = IntegerCaster.cast( config.get( Key.maxObjects ) ); + this.pool = new ConcurrentHashMap<>( maxObjects / 4 ); + this.softRefKeyMap = new ConcurrentHashMap<>( maxObjects / 4 ); this.referenceQueue = new ReferenceQueue<>(); logger.debug( "ConcurrentSoftReferenceStore({}) initialized with a max size of {}", provider.getName(), - config.getAsInteger( Key.maxObjects ) + maxObjects ); return this; @@ -139,7 +142,8 @@ public int flush() { * and eviction count. */ public synchronized void evict() { - if ( this.config.getAsInteger( Key.evictCount ) == 0 ) { + int evictCount = IntegerCaster.cast( this.config.get( Key.evictCount ) ); + if ( evictCount == 0 ) { return; } getPool() @@ -153,7 +157,7 @@ public synchronized void evict() { // Sort using the policy comparator .sorted( getPolicy().getComparator() ) // Check how many to evict according to the config count - .limit( this.config.getAsInteger( Key.evictCount ) ) + .limit( evictCount ) // Evict it & Log Stats .forEach( entry -> { logger.debug( @@ -340,7 +344,7 @@ public ICacheEntry get( Key key ) { .incrementHits() .touchLastAccessed(); // Is resetTimeoutOnAccess enabled? If so, jump up the creation time to increase the timeout - if ( this.config.getAsBoolean( Key.resetTimeoutOnAccess ) ) { + if ( BooleanCaster.cast( this.config.get( Key.resetTimeoutOnAccess ) ) ) { results.resetCreated(); } } diff --git a/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentStore.java b/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentStore.java index ec42f9ff4..c0bd751af 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentStore.java +++ b/src/main/java/ortus/boxlang/runtime/cache/store/ConcurrentStore.java @@ -28,6 +28,8 @@ import ortus.boxlang.runtime.cache.ICacheEntry; import ortus.boxlang.runtime.cache.filters.ICacheKeyFilter; import ortus.boxlang.runtime.cache.providers.ICacheProvider; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; @@ -66,12 +68,13 @@ public ConcurrentStore() { public IObjectStore init( ICacheProvider provider, IStruct config ) { this.provider = provider; this.config = config; - this.pool = new ConcurrentHashMap<>( config.getAsInteger( Key.maxObjects ) / 4 ); + int maxObject = IntegerCaster.cast( config.get( Key.maxObjects ) ); + this.pool = new ConcurrentHashMap<>( maxObject / 4 ); logger.debug( "ConcurrentStore({}) initialized with a max size of {}", provider.getName(), - config.getAsInteger( Key.maxObjects ) + maxObject ); return this; } @@ -125,7 +128,8 @@ public int flush() { * and eviction count. */ public synchronized void evict() { - if ( this.config.getAsInteger( Key.evictCount ) == 0 ) { + int evictCount = IntegerCaster.cast( this.config.get( Key.evictCount ) ); + if ( evictCount == 0 ) { return; } getPool().entrySet() @@ -136,7 +140,7 @@ public synchronized void evict() { // Exclude eternal objects from eviction .filter( entry -> !entry.getValue().isEternal() ) // Check how many to evict according to the config count - .limit( this.config.getAsInteger( Key.evictCount ) ) + .limit( evictCount ) // Evict it & Log Stats .forEach( entry -> { logger.debug( @@ -298,7 +302,7 @@ public ICacheEntry get( Key key ) { .incrementHits() .touchLastAccessed(); // Is resetTimeoutOnAccess enabled? If so, jump up the creation time to increase the timeout - if ( this.config.getAsBoolean( Key.resetTimeoutOnAccess ) ) { + if ( BooleanCaster.cast( this.config.get( Key.resetTimeoutOnAccess ) ) ) { results.resetCreated(); } } diff --git a/src/main/java/ortus/boxlang/runtime/cache/store/FileSystemStore.java b/src/main/java/ortus/boxlang/runtime/cache/store/FileSystemStore.java index 57659ffed..8f156fd5c 100644 --- a/src/main/java/ortus/boxlang/runtime/cache/store/FileSystemStore.java +++ b/src/main/java/ortus/boxlang/runtime/cache/store/FileSystemStore.java @@ -34,6 +34,8 @@ import ortus.boxlang.runtime.cache.ICacheEntry; import ortus.boxlang.runtime.cache.filters.ICacheKeyFilter; import ortus.boxlang.runtime.cache.providers.ICacheProvider; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; @@ -97,7 +99,7 @@ public IObjectStore init( ICacheProvider provider, IStruct config ) { logger.debug( "FileSystemStore({}) initialized with a max size of {}", provider.getName(), - config.getAsInteger( Key.maxObjects ) + IntegerCaster.cast( config.get( Key.maxObjects ) ) ); return this; } @@ -171,7 +173,8 @@ public int flush() { * and eviction count. */ public synchronized void evict() { - if ( this.config.getAsInteger( Key.evictCount ) == 0 ) { + int evictCount = IntegerCaster.cast( this.config.get( Key.evictCount ) ); + if ( evictCount == 0 ) { return; } getEntryStream() @@ -181,7 +184,7 @@ public synchronized void evict() { // Exclude eternal objects from eviction .filter( entry -> !entry.isEternal() ) // Check how many to evict according to the config count - .limit( this.config.getAsInteger( Key.evictCount ) ) + .limit( evictCount ) // Evict it & Log Stats .forEach( entry -> { logger.debug( @@ -370,7 +373,7 @@ public ICacheEntry get( Key key ) { .incrementHits() .touchLastAccessed(); // Is resetTimeoutOnAccess enabled? If so, jump up the creation time to increase the timeout - if ( this.config.getAsBoolean( Key.resetTimeoutOnAccess ) ) { + if ( BooleanCaster.cast( this.config.get( Key.resetTimeoutOnAccess ) ) ) { results.resetCreated(); } } diff --git a/src/main/java/ortus/boxlang/runtime/components/cache/Cache.java b/src/main/java/ortus/boxlang/runtime/components/cache/Cache.java index 35bd7e546..00674bd48 100644 --- a/src/main/java/ortus/boxlang/runtime/components/cache/Cache.java +++ b/src/main/java/ortus/boxlang/runtime/components/cache/Cache.java @@ -31,6 +31,7 @@ import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.Attempt; import ortus.boxlang.runtime.dynamic.ExpressionInterpreter; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.events.BoxEvent; @@ -166,10 +167,10 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod String variable = attributes.getAsString( Key._NAME ); String cacheName = attributes.getAsString( Key.cacheName ); String cacheDirectory = attributes.getAsString( Key.directory ); - Boolean useCache = attributes.getAsBoolean( Key.useCache ); - Double timespan = attributes.getAsDouble( Key.timespan ); - Double idleTime = attributes.getAsDouble( Key.idleTime ); - Boolean throwOnError = attributes.getAsBoolean( Key.throwOnError ); + Boolean useCache = BooleanCaster.cast( attributes.get( Key.useCache ) ); + Double timespan = DoubleCaster.cast( attributes.get( Key.timespan ) ); + Double idleTime = DoubleCaster.cast( attributes.get( Key.idleTime ) ); + Boolean throwOnError = BooleanCaster.cast( attributes.get( Key.throwOnError ) ); ICacheProvider cacheProvider = null; List namedCacheOps = List.of( CacheAction.GET, @@ -325,7 +326,7 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod result = castedAttempt.get(); } - if ( result instanceof String && attributes.getAsBoolean( Key.stripWhitespace ) ) { + if ( result instanceof String && BooleanCaster.cast( attributes.get( Key.stripWhitespace ) ) ) { result = StringCaster.cast( result ).trim(); } diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 6ea487aae..6772c7ec7 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1570,6 +1570,7 @@ public void testMixinsPublic() { @DisplayName( "class locator usage in static initializer" ) @Test + @Disabled public void testClassLocatorInStaticInitializer() { instance.executeSource( """ From 42286035c4705642f6c1186d17219a08d5569521 Mon Sep 17 00:00:00 2001 From: Jacob Beers Date: Thu, 19 Dec 2024 00:05:12 -0600 Subject: [PATCH 037/161] BL-848 fix class locator issues --- .../transformer/expression/BoxAccessTransformer.java | 7 +++---- .../expression/BoxBinaryOperationTransformer.java | 10 +++++----- .../BoxExpressionInvocationTransformer.java | 3 +-- .../transformer/expression/BoxNewTransformer.java | 12 ++++++++---- .../expression/BoxUnaryOperationTransformer.java | 7 +++---- .../transformer/statement/BoxAssertTransformer.java | 3 +-- .../statement/BoxBufferOutputTransformer.java | 3 +-- .../phase3/ClassLocatorInStaticInitializer.bx | 3 ++- src/test/java/TestCases/phase3/ClassTest.java | 1 - 9 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java index 2e12b21aa..1952cb74b 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java @@ -26,7 +26,6 @@ import org.objectweb.asm.tree.FieldInsnNode; import org.objectweb.asm.tree.InsnNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -89,7 +88,7 @@ public List transform( BoxNode node, TransformerContext contex // return javaExpr; List nodes = new ArrayList<>(); nodes.addAll( transpiler.transform( objectAccess.getContext(), TransformerContext.NONE, ReturnValueContext.VALUE ) ); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( accessKey ); nodes.add( new FieldInsnNode( Opcodes.GETSTATIC, @@ -116,7 +115,7 @@ public List transform( BoxNode node, TransformerContext contex } else { // BoxNode parent = ( BoxNode ) objectAccess.getParent(); List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( transpiler.transform( objectAccess.getContext(), context, ReturnValueContext.VALUE ) ); nodes.addAll( accessKey ); nodes.add( new FieldInsnNode( Opcodes.GETSTATIC, @@ -139,7 +138,7 @@ public List transform( BoxNode node, TransformerContext contex // This prolly won't work if a query column is passed as a second param that isn't the array && ! ( parent instanceof BoxArgument barg && barg.getParent() instanceof BoxFunctionInvocation bfun && bfun.getName().toLowerCase().contains( "array" ) ) ) { - nodes.add( 0, new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( 0, transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.add( new MethodInsnNode( Opcodes.INVOKEINTERFACE, Type.getInternalName( IBoxContext.class ), "unwrapQueryColumn", diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBinaryOperationTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBinaryOperationTransformer.java index c880b05a3..ca4b8b4be 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBinaryOperationTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBinaryOperationTransformer.java @@ -27,7 +27,6 @@ import org.objectweb.asm.tree.JumpInsnNode; import org.objectweb.asm.tree.LabelNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -189,7 +188,7 @@ public List transform( BoxNode node, TransformerContext contex generateBinaryMethodCallNodes( Elvis.class, Object.class, left, right ); case InstanceOf -> // "InstanceOf.invoke(${contextName},${left},${right})"; - generateBinaryMethodCallNodesWithContext( InstanceOf.class, Boolean.class, left, right ); + generateBinaryMethodCallNodesWithContext( transpiler, InstanceOf.class, Boolean.class, left, right ); case Contains -> // "Contains.invoke(${left},${right})"; generateBinaryMethodCallNodes( Contains.class, Boolean.class, left, right ); @@ -198,7 +197,7 @@ public List transform( BoxNode node, TransformerContext contex generateBinaryMethodCallNodes( NotContains.class, Boolean.class, left, right ); case CastAs -> // "CastAs.invoke(${contextName},${left},${right})"; - generateBinaryMethodCallNodesWithContext( CastAs.class, Object.class, left, right ); + generateBinaryMethodCallNodesWithContext( transpiler, CastAs.class, Object.class, left, right ); case BitwiseAnd -> // "BitwiseAnd.invoke(${left},${right})"; generateBinaryMethodCallNodes( BitwiseAnd.class, Number.class, left, right ); @@ -243,10 +242,11 @@ private static List generateBinaryMethodCallNodes( Class di } @Nonnull - private static List generateBinaryMethodCallNodesWithContext( Class dispatcher, Class returned, List left, + private static List generateBinaryMethodCallNodesWithContext( Transpiler transpiler, Class dispatcher, Class returned, + List left, List right ) { List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( left ); nodes.addAll( right ); nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxExpressionInvocationTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxExpressionInvocationTransformer.java index 9aed02ae0..245b4eb3c 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxExpressionInvocationTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxExpressionInvocationTransformer.java @@ -21,7 +21,6 @@ import org.objectweb.asm.Type; import org.objectweb.asm.tree.AbstractInsnNode; import org.objectweb.asm.tree.InsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.AsmTranspiler; @@ -47,7 +46,7 @@ public List transform( BoxNode node, TransformerContext contex List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); Type invokeType = getInvocationType( invocation.getExpr() ); diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxNewTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxNewTransformer.java index dc473e915..4eac27446 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxNewTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxNewTransformer.java @@ -26,7 +26,6 @@ import org.objectweb.asm.tree.FieldInsnNode; import org.objectweb.asm.tree.LdcInsnNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -51,9 +50,14 @@ public List transform( BoxNode node, TransformerContext contex BoxNew boxNew = ( BoxNew ) node; List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 2 ) ); + // nodes.add( new VarInsnNode( Opcodes.ALOAD, 2 ) ); + nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, + Type.getInternalName( ClassLocator.class ), + "getInstance", + Type.getMethodDescriptor( Type.getType( ClassLocator.class ) ), + false ) ); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.add( new LdcInsnNode( "" ) ); // TODO: how to set this? nodes.addAll( transpiler.transform( boxNew.getExpression(), TransformerContext.NONE, ReturnValueContext.VALUE ) ); nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, @@ -82,7 +86,7 @@ public List transform( BoxNode node, TransformerContext contex Type.getType( List.class ) ), false ) ); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( AsmHelper.callDynamicObjectInvokeConstructor( transpiler, boxNew.getArguments(), context ) ); diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxUnaryOperationTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxUnaryOperationTransformer.java index 6894926b2..505a3b90b 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxUnaryOperationTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxUnaryOperationTransformer.java @@ -23,7 +23,6 @@ import org.objectweb.asm.tree.InsnNode; import org.objectweb.asm.tree.LdcInsnNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -80,8 +79,8 @@ public List transform( BoxNode node, TransformerContext contex List nodes = new ArrayList<>(); // for non literals, we need to identify the key being incremented/decremented and the object it lives in (which may be a scope) if ( expr instanceof BoxIdentifier id && operator != BoxUnaryOperator.Not && operator != BoxUnaryOperator.Minus && operator != BoxUnaryOperator.Plus ) { - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( transpiler.createKey( id.getName() ) ); nodes.add( new InsnNode( Opcodes.ACONST_NULL ) ); nodes.add( new LdcInsnNode( true ) ); @@ -100,7 +99,7 @@ public List transform( BoxNode node, TransformerContext contex nodes.add( getMethodCallTemplateCompound( operation ) ); } else if ( expr instanceof BoxAccess objectAccess && operator != BoxUnaryOperator.Not && operator != BoxUnaryOperator.Minus && operator != BoxUnaryOperator.Plus ) { - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( transpiler.transform( objectAccess.getContext(), TransformerContext.NONE, ReturnValueContext.VALUE ) ); // DotAccess just uses the string directly, array access allows any expression> List accessKey; diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxAssertTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxAssertTransformer.java index fbe328719..fcd3cac8d 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxAssertTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxAssertTransformer.java @@ -22,7 +22,6 @@ import org.objectweb.asm.tree.AbstractInsnNode; import org.objectweb.asm.tree.InsnNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -44,7 +43,7 @@ public BoxAssertTransformer( Transpiler transpiler ) { public List transform( BoxNode node, TransformerContext context, ReturnValueContext returnContext ) throws IllegalStateException { BoxAssert boxAssert = ( BoxAssert ) node; List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.addAll( transpiler.transform( boxAssert.getExpression(), TransformerContext.RIGHT, ReturnValueContext.VALUE ) ); nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, Type.getInternalName( Assert.class ), diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxBufferOutputTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxBufferOutputTransformer.java index 95640b2a2..ea85adde8 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxBufferOutputTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/statement/BoxBufferOutputTransformer.java @@ -25,7 +25,6 @@ import org.objectweb.asm.tree.AbstractInsnNode; import org.objectweb.asm.tree.InsnNode; import org.objectweb.asm.tree.MethodInsnNode; -import org.objectweb.asm.tree.VarInsnNode; import ortus.boxlang.compiler.asmboxpiler.AsmHelper; import ortus.boxlang.compiler.asmboxpiler.Transpiler; @@ -47,7 +46,7 @@ public List transform( BoxNode node, TransformerContext contex BoxBufferOutput bufferOuput = ( BoxBufferOutput ) node; List nodes = new ArrayList<>(); - nodes.add( new VarInsnNode( Opcodes.ALOAD, 1 ) ); + nodes.addAll( transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes .addAll( transpiler.transform( bufferOuput.getExpression(), TransformerContext.NONE, ReturnValueContext.VALUE_OR_NULL ) ); nodes.add( new MethodInsnNode( Opcodes.INVOKEINTERFACE, diff --git a/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx index 286b5156a..3b75ac618 100644 --- a/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx +++ b/src/test/java/TestCases/phase3/ClassLocatorInStaticInitializer.bx @@ -1,6 +1,7 @@ class { static { - createObject( "java", "java.io.File" ); + static.a = new src.test.java.TestCases.phase3.ConcreteClass(); + static.x = createObject( "java", "java.io.File" ); final slashdir = createObject( "java", "java.io.File" ).separator; } } \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 6772c7ec7..6ea487aae 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1570,7 +1570,6 @@ public void testMixinsPublic() { @DisplayName( "class locator usage in static initializer" ) @Test - @Disabled public void testClassLocatorInStaticInitializer() { instance.executeSource( """ From 6c2ccef087688b1d17a3d9b10301a027b49ff52f Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Wed, 18 Dec 2024 21:16:03 +0100 Subject: [PATCH 038/161] BL-867 #resolve Show the right exception type when invoking dynamic objects and there are exceptions from a caused by --- .../interop/DynamicInteropService.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java index 42ea60492..346be769c 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java +++ b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java @@ -374,7 +374,12 @@ public static T invokeConstructor( IBoxContext context, Class targetClass } catch ( RuntimeException e ) { throw e; } catch ( Throwable e ) { - throw new BoxRuntimeException( "Error invoking constructor for class " + targetClass.getName(), e ); + throw new BoxRuntimeException( + "Error invoking constructor for class " + targetClass.getName() + + ". Caused by " + e.getMessage(), + e.getClass().getName(), + e + ); } } @@ -536,7 +541,12 @@ public static Object invoke( } catch ( RuntimeException e ) { throw e; } catch ( Throwable e ) { - throw new BoxRuntimeException( "Error invoking method " + methodName + "() for class " + targetClass.getName(), e ); + throw new BoxRuntimeException( + "Error invoking method " + methodName + "() for class " + targetClass.getName() + + ". Due to: " + e.getMessage(), + e.getClass().getName(), + e + ); } } @@ -578,7 +588,12 @@ public static Object invokeStatic( IBoxContext context, DynamicObject dynamicObj } catch ( RuntimeException e ) { throw e; } catch ( Throwable e ) { - throw new BoxRuntimeException( "Error invoking method " + methodName + "() for class " + targetClass.getName(), e ); + throw new BoxRuntimeException( + "Error invoking method " + methodName + "() for class " + targetClass.getName() + + ". Caused by " + e.getMessage(), + e.getClass().getName(), + e + ); } } @@ -657,7 +672,11 @@ public static Optional getField( Class targetClass, Object targetInst } catch ( RuntimeException e ) { throw e; } catch ( Throwable e ) { - throw new BoxRuntimeException( "Error getting field " + fieldName + " for class " + targetClass.getName(), e ); + throw new BoxRuntimeException( + "Error getting field " + fieldName + " for class " + targetClass.getName(), + e.getClass().getName(), + e + ); } } @@ -768,7 +787,11 @@ public static void setField( Class targetClass, Object targetInstance, String } catch ( RuntimeException e ) { throw e; } catch ( Throwable e ) { - throw new BoxRuntimeException( "Error setting field " + fieldName + " for class " + targetClass.getName(), e ); + throw new BoxRuntimeException( + "Error setting field " + fieldName + " for class " + targetClass.getName(), + e.getClass().getName(), + e + ); } } From 4bde88283712ad3203210b0b889defe3a6477c06 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Thu, 19 Dec 2024 15:18:15 -0500 Subject: [PATCH 039/161] recreated last async bug ColdBox --- src/test/java/TestCases/phase3/ClassTest.java | 120 +++++++++++------- .../TestCases/phase3/RelativeInstantiation.bx | 16 ++- .../java/TestCases/phase3/RelativeManager.bx | 12 ++ .../java/TestCases/phase3/tasks/Future.cfc | 15 +++ 4 files changed, 113 insertions(+), 50 deletions(-) create mode 100644 src/test/java/TestCases/phase3/RelativeManager.bx create mode 100644 src/test/java/TestCases/phase3/tasks/Future.cfc diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 6ea487aae..4bad4b160 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1339,24 +1339,44 @@ public void testDotExtends() { @Test public void testRelativeInstantiation() { + // @formatter:off instance.executeSource( """ - clazz = new src.test.java.TestCases.phase3.RelativeInstantiation(); - result = clazz.findSibling() - result2 = getMetadata( clazz.returnSibling() ) - """, context ); + clazz = new src.test.java.TestCases.phase3.RelativeInstantiation(); + result = clazz.findSibling() + result2 = getMetadata( clazz.returnSibling() ) + """, context ); + // @formatter:on assertThat( variables.get( result ) ).isEqualTo( "bar" ); - assertThat( variables.getAsStruct( Key.of("result2") ).getAsString(Key._NAME ) ).isEqualTo("src.test.java.TestCases.phase3.FindMe"); + assertThat( variables.getAsStruct( Key.of( "result2" ) ).getAsString( Key._NAME ) ).isEqualTo( "src.test.java.TestCases.phase3.FindMe" ); + } + + @Disabled( "Brad Fix this pretty please!!" ) + @Test + public void testRelativeInstantiationWithLoop() { + // @formatter:off + instance.executeSource( + """ + manager = new src.test.java.TestCases.phase3.RelativeManager() + data = [1,2,3,4,5] + result = manager.allApply( data ) + """, context ); + // @formatter:on + Array target = variables.getAsArray( result ); + System.out.println( target ); } @Test public void testRelativeSelfInstantiation() { + // @formatter:off instance.executeSource( """ - clazz = new src.test.java.TestCases.phase3.RelativeSelfInstantiation(); - result2 = getMetadata( clazz.clone() ) - """, context ); - assertThat( variables.getAsStruct( Key.of("result2") ).getAsString(Key._NAME ) ).isEqualTo("src.test.java.TestCases.phase3.RelativeSelfInstantiation"); + clazz = new src.test.java.TestCases.phase3.RelativeSelfInstantiation(); + result2 = getMetadata( clazz.clone() ) + """, context ); + // @formatter:on + assertThat( variables.getAsStruct( Key.of( "result2" ) ).getAsString( Key._NAME ) ) + .isEqualTo( "src.test.java.TestCases.phase3.RelativeSelfInstantiation" ); } @Test @@ -1538,19 +1558,22 @@ public void testUDFClassEnclosingClassReference() { instance.executeSource( """ - import bx:src.test.java.TestCases.phase3.PropertyTestCF as brad; - b = new brad() - outerClass = b.$bx.$class; - innerClass = b.init.getClass(); - innerClassesOuterClass = b.init.getClass().getEnclosingClass(); - println(outerclass) - println(innerClass) - """, + import bx:src.test.java.TestCases.phase3.PropertyTestCF as brad; + b = new brad() + outerClass = b.$bx.$class; + innerClass = b.init.getClass(); + innerClassesOuterClass = b.init.getClass().getEnclosingClass(); + println(outerclass) + println(innerClass) + """, context ); - assertThat(((Class)variables.get( "outerClass" )).getName() ).isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc" ); - assertThat(((Class)variables.get( "innerClassesOuterClass" )).getName() ).isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc" ); - assertThat(((Class)variables.get( "innerClass" )).getName() ).isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc$Func_init" ); - assertThat( variables.get( "outerClass" ) ).isEqualTo( variables.get("innerClassesOuterClass") ); + assertThat( ( ( Class ) variables.get( "outerClass" ) ).getName() ) + .isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc" ); + assertThat( ( ( Class ) variables.get( "innerClassesOuterClass" ) ).getName() ) + .isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc" ); + assertThat( ( ( Class ) variables.get( "innerClass" ) ).getName() ) + .isEqualTo( "boxgenerated.boxclass.src.test.java.testcases.phase3.Propertytestcf$cfc$Func_init" ); + assertThat( variables.get( "outerClass" ) ).isEqualTo( variables.get( "innerClassesOuterClass" ) ); } @DisplayName( "mixins should be public" ) @@ -1559,15 +1582,14 @@ public void testMixinsPublic() { instance.executeSource( """ - mt = new src.test.java.TestCases.phase3.MixinTest() - mt.includeMixin(); - result = mt.mixed() - """, + mt = new src.test.java.TestCases.phase3.MixinTest() + mt.includeMixin(); + result = mt.mixed() + """, context ); assertThat( variables.get( "result" ) ).isEqualTo( "mixed up" ); } - @DisplayName( "class locator usage in static initializer" ) @Test public void testClassLocatorInStaticInitializer() { @@ -1583,34 +1605,34 @@ public void testClassLocatorInStaticInitializer() { public void testPropertiesNotInheritedInMetadata() { instance.executeSource( """ - c = new src.test.java.TestCases.phase3.PropertiesNotInheritedInMetadata(); - result = getMetaData( c ) - result2 = c.$bx.meta - """, + c = new src.test.java.TestCases.phase3.PropertiesNotInheritedInMetadata(); + result = getMetaData( c ) + result2 = c.$bx.meta + """, context ); - + // Legacy metadata Array childProps = variables.getAsStruct( Key.of( "result" ) ).getAsArray( Key.properties ); - assertThat( childProps ).hasSize(2); - assertThat( ((IStruct)childProps.get(0)).get( Key._NAME ) ).isEqualTo( "childProperty" ); - assertThat( ((IStruct)childProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); - - Array parentProps = variables.getAsStruct( Key.of( "result" ) ).getAsStruct(Key._EXTENDS).getAsArray( Key.properties ); - assertThat( parentProps ).hasSize(2); - assertThat( ((IStruct)parentProps.get(0)).get( Key._NAME ) ).isEqualTo( "parentProperty" ); - assertThat( ((IStruct)parentProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + assertThat( childProps ).hasSize( 2 ); + assertThat( ( ( IStruct ) childProps.get( 0 ) ).get( Key._NAME ) ).isEqualTo( "childProperty" ); + assertThat( ( ( IStruct ) childProps.get( 1 ) ).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + Array parentProps = variables.getAsStruct( Key.of( "result" ) ).getAsStruct( Key._EXTENDS ).getAsArray( Key.properties ); + assertThat( parentProps ).hasSize( 2 ); + assertThat( ( ( IStruct ) parentProps.get( 0 ) ).get( Key._NAME ) ).isEqualTo( "parentProperty" ); + assertThat( ( ( IStruct ) parentProps.get( 1 ) ).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); // $bx metadata - childProps = variables.getAsStruct( Key.of( Key.of("result2") ) ).getAsArray( Key.properties ); - assertThat( childProps ).hasSize(2); - assertThat( ((IStruct)childProps.get(0)).get( Key._NAME ) ).isEqualTo( "childProperty" ); - assertThat( ((IStruct)childProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); - - parentProps = variables.getAsStruct( Key.of( Key.of("result2")) ).getAsStruct(Key._EXTENDS).getAsArray( Key.properties ); - assertThat( parentProps ).hasSize(2); - assertThat( ((IStruct)parentProps.get(0)).get( Key._NAME ) ).isEqualTo( "parentProperty" ); - assertThat( ((IStruct)parentProps.get(1)).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); - + childProps = variables.getAsStruct( Key.of( Key.of( "result2" ) ) ).getAsArray( Key.properties ); + assertThat( childProps ).hasSize( 2 ); + assertThat( ( ( IStruct ) childProps.get( 0 ) ).get( Key._NAME ) ).isEqualTo( "childProperty" ); + assertThat( ( ( IStruct ) childProps.get( 1 ) ).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + + parentProps = variables.getAsStruct( Key.of( Key.of( "result2" ) ) ).getAsStruct( Key._EXTENDS ).getAsArray( Key.properties ); + assertThat( parentProps ).hasSize( 2 ); + assertThat( ( ( IStruct ) parentProps.get( 0 ) ).get( Key._NAME ) ).isEqualTo( "parentProperty" ); + assertThat( ( ( IStruct ) parentProps.get( 1 ) ).get( Key._NAME ) ).isEqualTo( "sharedProperty" ); + } } diff --git a/src/test/java/TestCases/phase3/RelativeInstantiation.bx b/src/test/java/TestCases/phase3/RelativeInstantiation.bx index 4f87a8ebb..db495c73d 100644 --- a/src/test/java/TestCases/phase3/RelativeInstantiation.bx +++ b/src/test/java/TestCases/phase3/RelativeInstantiation.bx @@ -1,8 +1,22 @@ class { + + property name="name"; + + RelativeInstantiation function init( string item = "nothing" ){ + variables.name = arguments.item; + return this; + } + function findSibling() { return new FindMe().foo(); } function returnSibling() { return new FindMe(); } -} \ No newline at end of file + + function allApply( items ){ + return arguments + .items + .map( item => new RelativeInstantiation( item ) ) + } +} diff --git a/src/test/java/TestCases/phase3/RelativeManager.bx b/src/test/java/TestCases/phase3/RelativeManager.bx new file mode 100644 index 000000000..393a1af89 --- /dev/null +++ b/src/test/java/TestCases/phase3/RelativeManager.bx @@ -0,0 +1,12 @@ +class{ + + + any function allApply(){ + return newFuture().allApply( argumentCollection = arguments ); + } + + RelativeInstantiation function newFuture(){ + return new tasks.Future(); + } + +} diff --git a/src/test/java/TestCases/phase3/tasks/Future.cfc b/src/test/java/TestCases/phase3/tasks/Future.cfc new file mode 100644 index 000000000..d1dc5a21e --- /dev/null +++ b/src/test/java/TestCases/phase3/tasks/Future.cfc @@ -0,0 +1,15 @@ +component { + + property name="name"; + + Future function init( string item = "nothing" ){ + variables.name = arguments.item; + return this; + } + + function allApply( items ){ + return arguments + .items + .map( item => new Future( item ) ) + } +} From 15e20465dde4943e68f013fa17ba1bd33438365e Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 19 Dec 2024 17:10:47 -0600 Subject: [PATCH 040/161] BL-870 --- .../ortus/boxlang/runtime/types/Struct.java | 33 ++++--- .../java/TestCases/phase1/CoreLangTest.java | 91 +++++++++++++------ .../java/TestCases/phase1/TestReturnType.bx | 6 ++ .../java/TestCases/phase1/TestReturnType.bxm | 8 ++ .../boxlang/runtime/types/StructTest.java | 17 ++++ 5 files changed, 114 insertions(+), 41 deletions(-) create mode 100644 src/test/java/TestCases/phase1/TestReturnType.bx create mode 100644 src/test/java/TestCases/phase1/TestReturnType.bxm diff --git a/src/main/java/ortus/boxlang/runtime/types/Struct.java b/src/main/java/ortus/boxlang/runtime/types/Struct.java index 006cf2d75..01c5155f3 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Struct.java +++ b/src/main/java/ortus/boxlang/runtime/types/Struct.java @@ -421,7 +421,7 @@ public boolean containsValue( Object value ) { @Override public Object get( Object key ) { if ( key instanceof Key keyKey ) { - return unWrapNull( + return unWrapNullInternal( isCaseSensitive() ? wrapped.get( keySet().stream().filter( k -> KeyCaster.cast( k ).equalsWithCase( keyKey ) ).findFirst().orElse( Key.EMPTY ) ) : wrapped.get( keyKey ) @@ -442,7 +442,7 @@ public Object get( Object key ) { */ public Object get( String key ) { Key keyObj = Key.of( key ); - return unWrapNull( + return unWrapNullInternal( isCaseSensitive() ? wrapped.get( keySet().stream().filter( k -> KeyCaster.cast( k ).equalsWithCase( keyObj ) ).findFirst().orElse( Key.EMPTY ) ) : wrapped.get( keyObj ) @@ -459,10 +459,10 @@ public Object get( String key ) { */ public Object getOrDefault( Key key, Object defaultValue ) { return isCaseSensitive() - ? unWrapNull( + ? unWrapNullInternal( wrapped.getOrDefault( keySet().stream().filter( k -> KeyCaster.cast( k ).equalsWithCase( key ) ).findFirst().orElse( Key.EMPTY ), defaultValue ) ) - : unWrapNull( wrapped.getOrDefault( key, defaultValue ) ); + : unWrapNullInternal( wrapped.getOrDefault( key, defaultValue ) ); } @@ -676,7 +676,7 @@ public Set keySet() { @Override public Collection values() { return wrapped.values().stream() - .map( entry -> unWrapNull( entry ) ) + .map( entry -> unWrapNullInternal( entry ) ) .collect( Collectors.toList() ); } @@ -686,7 +686,7 @@ public Collection values() { @Override public Set> entrySet() { return wrapped.entrySet().stream() - .map( entry -> new SimpleEntry<>( entry.getKey(), unWrapNull( entry.getValue() ) ) ) + .map( entry -> new SimpleEntry<>( entry.getKey(), unWrapNullInternal( entry.getValue() ) ) ) .collect( Collectors.toCollection( LinkedHashSet::new ) ); } @@ -868,7 +868,7 @@ public Object dereference( IBoxContext context, Key key, Boolean safe ) { String.format( "The key [%s] was not found in the struct. Valid keys are (%s)", key.getName(), getKeysAsStrings() ), this ); } - return unWrapNull( value ); + return unWrapNullInternal( value ); } /** @@ -1011,14 +1011,25 @@ public List getKeysAsStrings() { * * @return The unwrapped value which can be null */ - @SuppressWarnings( "unchecked" ) public static Object unWrapNull( Object value ) { if ( value instanceof NullValue ) { return null; } - return value instanceof SoftReference - ? ( ( SoftReference ) value ).get() - : value; + return value; + } + + /** + * Unwrap null values from the NullValue class + * + * @param value The value to unwrap + * + * @return The unwrapped value which can be null + */ + protected Object unWrapNullInternal( Object value ) { + if ( value instanceof NullValue ) { + return null; + } + return isSoftReferenced() && value instanceof SoftReference sr ? sr.get() : value; } /** diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index b6f1e032a..eba94afac 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; +import java.lang.ref.SoftReference; import java.math.BigInteger; import java.net.http.HttpRequest.BodyPublisher; import java.util.Comparator; @@ -4108,21 +4109,22 @@ public void testBadWhitespaceAfterWordOperator() { @DisplayName( "nested try catch with specific catch" ) @Test public void testNestedTryCatchWithSpecificCatch() { - // @formatter:off + // @formatter:on instance.executeSource( """ - result = "default"; - try { - try { - throw( type = "Different", message = "boom" ); - } catch ( Specific e ) { - result = "specific"; - } - } catch ( any e ) { - result = "general"; - } - """, + result = "default"; + try { + try { + throw( type = "Different", message = "boom" ); + } catch ( Specific e ) { + result = "specific"; + } + } catch ( any e ) { + result = "general"; + } + """, context ); + // @formatter:off assertThat( variables.get( result ) ).isEqualTo( "general" ); } @@ -4140,26 +4142,27 @@ public void testComponentAttributeName() { @DisplayName( "It still sets variables in the local scope even if they are set to null" ) @Test public void testNullStillInLocalScope() { - // @formatter:off + // @formatter:on instance.executeSource( - """ - function returnsNull() { - return; - } + """ + function returnsNull() { + return; + } - function doesStuff() { - var inner = returnsNull(); - if ( !isNull( inner ) ) { - return inner; - } - inner = "local value leaked to variables"; - return "set this time"; - } - result = doesStuff(); - result = doesStuff(); - """, - context, BoxSourceType.CFSCRIPT ); + function doesStuff() { + var inner = returnsNull(); + if ( !isNull( inner ) ) { + return inner; + } + inner = "local value leaked to variables"; + return "set this time"; + } + result = doesStuff(); + result = doesStuff(); + """, + context, BoxSourceType.CFSCRIPT ); assertThat( variables.getAsString( result ) ).isEqualTo( "set this time" ); + // @formatter:off } @@ -4197,6 +4200,34 @@ public void testDumpOrder() { writeDump( var=Duration.ofHours(2).plusMinutes(30), label="Duration" ); """, context ); - // @formatter:on + // @formatter:on + } + + @Test + public void testReturnType() { + // @formatter:off + instance.executeSource( + """ + c = new src.test.java.TestCases.phase1.TestReturnType() + result = c.data; + """, + context ); + // @formatter:on + assertThat( variables.get( result ) ).isInstanceOf( IStruct.class ); + } + + @Test + public void testSoftRef() { + // @formatter:off + instance.executeSource( + """ + result = createObject( "java", "java.lang.ref.SoftReference" ).init( + "testr" + ); + """, + context ); + // @formatter:on + assertThat( variables.get( result ) ).isInstanceOf( SoftReference.class ); } + } diff --git a/src/test/java/TestCases/phase1/TestReturnType.bx b/src/test/java/TestCases/phase1/TestReturnType.bx new file mode 100644 index 000000000..7d2000a7b --- /dev/null +++ b/src/test/java/TestCases/phase1/TestReturnType.bx @@ -0,0 +1,6 @@ +class { + + include 'TestReturnType.bxm'; + this.data = getStruct(); + +} \ No newline at end of file diff --git a/src/test/java/TestCases/phase1/TestReturnType.bxm b/src/test/java/TestCases/phase1/TestReturnType.bxm new file mode 100644 index 000000000..abddbefbd --- /dev/null +++ b/src/test/java/TestCases/phase1/TestReturnType.bxm @@ -0,0 +1,8 @@ + + struct function getStruct() { + return { + foo = "bar", + baz = "bum" + }; + } + \ No newline at end of file diff --git a/src/test/java/ortus/boxlang/runtime/types/StructTest.java b/src/test/java/ortus/boxlang/runtime/types/StructTest.java index d8d89f896..b84293097 100644 --- a/src/test/java/ortus/boxlang/runtime/types/StructTest.java +++ b/src/test/java/ortus/boxlang/runtime/types/StructTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.ref.SoftReference; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -297,4 +298,20 @@ void testCanUseJavaObjectsAsKeys() { assertThat( variables.getAsArray( Key.of( "values" ) ) ).containsExactly( "test", "test2" ); } + @DisplayName( "Can store soft reference" ) + @Test + void testCanStoreSoftReference() { + // @formatter:off + instance.executeSource( + """ + myStr = {} + myStr.softRef = createObject( "java", "java.lang.ref.SoftReference" ).init( + "testr" + ); + """, + context ); + // @formatter:on + assertThat( variables.getAsStruct( Key.of( "myStr" ) ).get( Key.of( "softRef" ) ) ).isInstanceOf( SoftReference.class ); + } + } From 94aa1efdbb33efc771bd784de509f76b5b635a6a Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 19 Dec 2024 23:36:57 -0600 Subject: [PATCH 041/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 67 +- .../select/expression/SQLCountFunction.java | 13 + .../sql/select/expression/SQLFunction.java | 11 +- .../operation/SQLBinaryOperation.java | 12 +- .../compiler/toolchain/SQLVisitor.java | 129 ++-- .../boxlang/runtime/jdbc/QueryParameter.java | 5 +- .../runtime/jdbc/qoq/QoQExecutionService.java | 54 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 17 +- .../jdbc/qoq/functions/aggregate/Max.java | 1 + .../jdbc/qoq/functions/aggregate/Min.java | 1 + .../jdbc/qoq/functions/scalar/IsNull.java | 6 +- .../ortus/boxlang/runtime/scopes/Key.java | 1 + .../runtime/types/QueryColumnType.java | 2 + src/test/java/external/TestBoxTest.java | 146 ++++ .../java/external/specs/QoQAggregateTest.cfc | 65 ++ src/test/java/external/specs/QoQBasicTest.cfc | 641 ++++++++++++++++++ .../external/specs/QoQCountStructureTest.cfc | 92 +++ .../specs/QoQDistinctEdgeCaseTest.cfc | 34 + .../external/specs/QoQDistinctSortTest.cfc | 27 + .../java/external/specs/QoQDupeSelectTest.cfc | 49 ++ .../java/external/specs/QoQLists2Test.cfc_ | 281 ++++++++ .../java/external/specs/QoQListsTest.cfc_ | 266 ++++++++ .../java/external/specs/QoQNullParamTest.cfc | 148 ++++ .../external/specs/QoQOrderByEdgeCaseTest.cfc | 81 +++ .../ortus/boxlang/compiler/QoQParseTest.java | 90 ++- src/test/resources/box.json | 8 + 26 files changed, 2125 insertions(+), 122 deletions(-) create mode 100644 src/test/java/external/TestBoxTest.java create mode 100644 src/test/java/external/specs/QoQAggregateTest.cfc create mode 100644 src/test/java/external/specs/QoQBasicTest.cfc create mode 100644 src/test/java/external/specs/QoQCountStructureTest.cfc create mode 100644 src/test/java/external/specs/QoQDistinctEdgeCaseTest.cfc create mode 100644 src/test/java/external/specs/QoQDistinctSortTest.cfc create mode 100644 src/test/java/external/specs/QoQDupeSelectTest.cfc create mode 100644 src/test/java/external/specs/QoQLists2Test.cfc_ create mode 100644 src/test/java/external/specs/QoQListsTest.cfc_ create mode 100644 src/test/java/external/specs/QoQNullParamTest.cfc create mode 100644 src/test/java/external/specs/QoQOrderByEdgeCaseTest.cfc create mode 100644 src/test/resources/box.json diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 63a5e72be..8226212f5 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -211,65 +211,32 @@ drop_stmt: )? any_name ; -/* - SQLite understands the following binary operators, in order from highest to lowest precedence: - || - * / % - + - - << >> & | - < <= > >= - = == != <> IS IS NOT IS DISTINCT FROM IS NOT DISTINCT FROM IN LIKE GLOB MATCH REGEXP - AND - OR - */ +predicate: + expr (LT | LT_EQ | GT | GT_EQ) expr + | expr (ASSIGN | EQ | NOT_EQ1 | NOT_EQ2 | IS_ NOT_ | IS_ | LIKE_) expr + | predicate AND_ predicate + | predicate OR_ predicate + | expr NOT_? LIKE_ expr (ESCAPE_ expr)? + | expr IS_ NOT_? expr + | expr NOT_? BETWEEN_ expr AND_ expr + | expr NOT_? IN_ (OPEN_PAR (expr ( COMMA expr)*)? CLOSE_PAR | subquery_no_alias) +; + expr: literal_value | BIND_PARAMETER - //| ((schema_name DOT)? table_name DOT)? column_name | (table_name DOT)? column_name | unary_operator expr | expr PIPE2 expr | expr ( STAR | DIV | MOD) expr | expr (PLUS | MINUS) expr - // | expr ( LT2 | GT2 | AMP | PIPE) expr - | expr ( LT | LT_EQ | GT | GT_EQ) expr - | expr ( - ASSIGN - | EQ - | NOT_EQ1 - | NOT_EQ2 - | IS_ NOT_ - | IS_ - // | IS_ NOT_? DISTINCT_ FROM_ - // | IN_ - | LIKE_ - // | GLOB_ - // | MATCH_ - // | REGEXP_ - ) expr - | expr AND_ expr - | expr OR_ expr // Special handling of cast to allow cast( foo as number) | CAST_ OPEN_PAR expr AS_ (name | STRING_LITERAL) CLOSE_PAR // special handling of convert to allow convert( foo, number ) or convert( foo, 'number' ) | CONVERT_ OPEN_PAR expr COMMA (name | STRING_LITERAL) CLOSE_PAR - | function_name OPEN_PAR ((DISTINCT_? expr ( COMMA expr)*) | STAR)? CLOSE_PAR // filter_clause? over_clause? + | function_name OPEN_PAR ((DISTINCT_? ALL_? expr ( COMMA expr)*) | STAR)? CLOSE_PAR // filter_clause? over_clause? | OPEN_PAR expr CLOSE_PAR - //| OPEN_PAR expr (COMMA expr)* CLOSE_PAR - // | expr COLLATE_ collation_name - | expr NOT_? LIKE_ expr (ESCAPE_ expr)? - | expr IS_ NOT_? expr - | expr NOT_? BETWEEN_ expr AND_ expr - | expr NOT_? IN_ ( - // OPEN_PAR (select_stmt | expr ( COMMA expr)*)? CLOSE_PAR - OPEN_PAR (expr ( COMMA expr)*)? CLOSE_PAR - | subquery_no_alias - // | ( schema_name DOT)? table_name - // | (schema_name DOT)? table_function_name OPEN_PAR (expr (COMMA expr)*)? CLOSE_PAR - ) - // | ((NOT_)? EXISTS_)? OPEN_PAR select_stmt CLOSE_PAR | case_expr - // | raise_function ; case_expr: @@ -277,7 +244,7 @@ case_expr: ; case_when_then: - WHEN_ when_expr = expr THEN_ then_expr = expr + WHEN_ (when_expr = expr | when_predicate = predicate) THEN_ then_expr = expr ; raise_function: @@ -369,9 +336,9 @@ join: select_core: SELECT_ top? (DISTINCT_ /*| ALL_*/)? result_column (COMMA result_column)* ( FROM_ (table_or_subquery (COMMA table_or_subquery)* | join_clause) - )? (WHERE_ whereExpr = expr)? ( + )? (WHERE_ whereExpr = predicate)? ( GROUP_ BY_ groupByExpr += expr (COMMA groupByExpr += expr)* ( - HAVING_ havingExpr = expr + HAVING_ havingExpr = predicate )? )? limit_stmt? //(WINDOW_ window_name AS_ window_defn ( COMMA window_name AS_ window_defn)*)? @@ -426,7 +393,7 @@ join_operator: ; join_constraint: - ON_ expr + ON_ predicate // | USING_ OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR ; @@ -840,7 +807,7 @@ savepoint_name: ; table_alias: - any_name + IDENTIFIER ; transaction_name: diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java index e38b3c3cf..0fbf42466 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.compiler.ast.sql.select.expression; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -105,7 +106,19 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { */ public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { // TODO: handle distinct + + // If this is count(*), then we just return the number of intersections with no regard to whether there are nulls + if ( getArguments().get( 0 ) instanceof SQLStarExpression ) { + return intersections.size(); + } + // Count the non-null values var values = buildAggregateValues( QoQExec, intersections, getArguments().get( 0 ) ); + + // Distinct values + if ( isDistinct() ) { + // This may need a more robust comparison, we'll see. + values = Arrays.stream( values ).distinct().toArray(); + } return values.length; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java index 56e6e26d6..5b540aabd 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java @@ -146,13 +146,20 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { * Evaluate the expression aginst a partition of data */ public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { - if ( intersections.isEmpty() ) { + if ( intersections.isEmpty() && isAggregate() ) { return null; } QoQFunction function = QoQFunctionService.getFunction( name ); if ( function.isAggregate() ) { + List values = arguments.stream().map( a -> buildAggregateValues( QoQExec, intersections, a ) ).toList(); + // if all arrays in the list are empty, return null + if ( values.stream().allMatch( v -> v.length == 0 ) ) { + return null; + } return function.invokeAggregate( - arguments.stream().map( a -> buildAggregateValues( QoQExec, intersections, a ) ).toList(), arguments ); + values, + arguments + ); } else { return function.invoke( arguments.stream().map( a -> a.evaluateAggregate( QoQExec, intersections ) ).toList(), arguments ); } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java index 791bfbced..879654822 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java @@ -438,10 +438,12 @@ private void ensureBooleanOperands( QoQSelectExecution QoQExec ) { // These checks may or may not work. If we can't get away with this, then we can boolean cast the values // but SQL doesn't really have the same concept of truthiness and mostly expects to always get booleans from boolean columns or boolean expressions if ( !left.isBoolean( QoQExec ) ) { - throw new BoxRuntimeException( "Left side of a boolean [" + operator.getSymbol() + "] operation must be a boolean expression or bit column" ); + throw new BoxRuntimeException( "Left side of a boolean [" + operator.getSymbol() + "] operation must be a boolean expression or bit column. It is [" + + left.getClass().getName() + "]" ); } if ( !right.isBoolean( QoQExec ) ) { - throw new BoxRuntimeException( "Right side of a boolean [" + operator.getSymbol() + "] operation must be a boolean expression or bit column" ); + throw new BoxRuntimeException( "Right side of a boolean [" + operator.getSymbol() + + "] operation must be a boolean expression or bit column. It is [" + right.getClass().getName() + "]" ); } } @@ -450,10 +452,12 @@ private void ensureBooleanOperands( QoQSelectExecution QoQExec ) { */ private void ensureNumericOperands( QoQSelectExecution QoQExec ) { if ( !left.isNumeric( QoQExec ) ) { - throw new BoxRuntimeException( "Left side of a math [" + operator.getSymbol() + "] operation must be a numeric expression or numeric column" ); + throw new BoxRuntimeException( "Left side of a math [" + operator.getSymbol() + + "] operation must be a numeric expression or numeric column. It is [" + left.getClass().getName() + "]" ); } if ( !right.isNumeric( QoQExec ) ) { - throw new BoxRuntimeException( "Right side of a math [" + operator.getSymbol() + "] operation must be a numeric expression or numeric column" ); + throw new BoxRuntimeException( "Right side of a math [" + operator.getSymbol() + + "] operation must be a numeric expression or numeric column. It is [" + right.getClass().getName() + "]" ); } } diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index 60d92efda..719ffadc9 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -1,7 +1,9 @@ package ortus.boxlang.compiler.toolchain; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.antlr.v4.runtime.tree.TerminalNode; @@ -45,6 +47,7 @@ import ortus.boxlang.parser.antlr.SQLGrammar.Literal_valueContext; import ortus.boxlang.parser.antlr.SQLGrammar.Ordering_termContext; import ortus.boxlang.parser.antlr.SQLGrammar.ParseContext; +import ortus.boxlang.parser.antlr.SQLGrammar.PredicateContext; import ortus.boxlang.parser.antlr.SQLGrammar.Result_columnContext; import ortus.boxlang.parser.antlr.SQLGrammar.Select_coreContext; import ortus.boxlang.parser.antlr.SQLGrammar.Select_stmtContext; @@ -64,7 +67,7 @@ public class SQLVisitor extends SQLGrammarBaseVisitor { private final SQLParser tools; private int bindCount = 0; - private int tableIndex = 0; + private Set tableIndex = new HashSet(); public SQLVisitor( SQLParser tools ) { this.tools = tools; @@ -187,6 +190,8 @@ public BoxNode visitSelect_stmt( Select_stmtContext ctx ) { */ @Override public SQLSelect visitSelect_core( Select_coreContext ctx ) { + + tableIndex = new HashSet(); var pos = tools.getPosition( ctx ); var src = tools.getSourceText( ctx ); @@ -230,7 +235,7 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { } if ( ctx.whereExpr != null ) { - where = visitExpr( ctx.whereExpr, table, joins ); + where = visitPredicate( ctx.whereExpr, table, joins ); } // group by @@ -242,7 +247,7 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { // having if ( ctx.havingExpr != null ) { - having = visitExpr( ctx.havingExpr, table, joins ); + having = visitPredicate( ctx.havingExpr, table, joins ); } // Do this after all joins above so we know the tables available to us @@ -306,7 +311,7 @@ public List buildJoins( SQLGrammar.Join_clauseContext ctx, SQLTable tab SQLJoin thisJoin = new SQLJoin( type, joinTable, null, pos, src ); joins.add( thisJoin ); if ( hasOn ) { - thisJoin.setOn( visitExpr( joinCtx.join_constraint().expr(), table, joins ) ); + thisJoin.setOn( visitPredicate( joinCtx.join_constraint().predicate(), table, joins ) ); } } return joins; @@ -353,7 +358,8 @@ public SQLTableVariable visitTable( TableContext ctx ) { alias = unwrapBracket( ctx.table_alias().getText() ); } - return new SQLTableVariable( schema, name, alias, tableIndex++, pos, src ); + tableIndex.add( name.toLowerCase() ); + return new SQLTableVariable( schema, name, alias, tableIndex.size() - 1, pos, src ); } /** @@ -370,7 +376,8 @@ public SQLTableSubQuery visitSubquery( SubqueryContext ctx ) { var src = tools.getSourceText( ctx ); SQLSelectStatement select = ( SQLSelectStatement ) new SQLVisitor( tools ).visit( ctx.select_stmt() ); - return new SQLTableSubQuery( select, ctx.table_alias().getText(), tableIndex++, pos, src ); + tableIndex.add( "subquery " + tableIndex.size() ); + return new SQLTableSubQuery( select, ctx.table_alias().getText(), tableIndex.size() - 1, pos, src ); } /** @@ -456,18 +463,6 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j return new SQLColumn( tableRef, unwrapBracket( ctx.column_name().getText() ), pos, src ); } else if ( ctx.literal_value() != null ) { return ( SQLExpression ) visit( ctx.literal_value() ); - } else if ( ctx.EQ() != null || ctx.ASSIGN() != null || ctx.IS_() != null ) { - // IS vs IS NOT - SQLBinaryOperator operator = ctx.NOT_() != null ? SQLBinaryOperator.NOTEQUAL : SQLBinaryOperator.EQUAL; - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), operator, pos, src, table, joins ); - } else if ( ctx.BETWEEN_() != null ) { - return new SQLBetweenOperation( visitExpr( ctx.expr( 0 ), table, joins ), visitExpr( ctx.expr( 1 ), table, joins ), - visitExpr( ctx.expr( 2 ), table, joins ), ctx.NOT_() != null, pos, src ); - } else if ( ctx.AND_() != null ) { - // Needs to run AFTER between checks - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.AND, pos, src, table, joins ); - } else if ( ctx.OR_() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.OR, pos, src, table, joins ); } else if ( ctx.CAST_() != null || ctx.CONVERT_() != null ) { // CAST( expr AS type ) Key functionName = ctx.CONVERT_() != null ? Key.convert : Key.cast; @@ -514,6 +509,64 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j name = ctx.BIND_PARAMETER().getText().substring( 1 ); } return new SQLParam( name, index, pos, src ); + } else if ( ctx.PIPE2() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.CONCAT, pos, src, table, joins ); + } else if ( ctx.STAR() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MULTIPLY, pos, src, table, joins ); + } else if ( ctx.DIV() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.DIVIDE, pos, src, table, joins ); + } else if ( ctx.MOD() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MODULO, pos, src, table, joins ); + } else if ( ctx.PLUS() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.PLUS, pos, src, table, joins ); + } else if ( ctx.MINUS() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MINUS, pos, src, table, joins ); + } else if ( ctx.OPEN_PAR() != null ) { + // Needs to run AFTER function and IN checks + return new SQLParenthesis( visitExpr( ctx.expr( 0 ), table, joins ), pos, src ); + } else if ( ctx.case_expr() != null ) { + return visitCase( ctx.case_expr(), table, joins ); + } else if ( ctx.unary_operator() != null ) { + SQLUnaryOperator op; + if ( ctx.unary_operator().BANG() != null ) { + op = SQLUnaryOperator.NOT; + } else if ( ctx.unary_operator().MINUS() != null ) { + op = SQLUnaryOperator.MINUS; + } else if ( ctx.unary_operator().PLUS() != null ) { + op = SQLUnaryOperator.PLUS; + } else { + throw new UnsupportedOperationException( "Unimplemented unary operator: " + ctx.unary_operator().getText() ); + } + return new SQLUnaryOperation( visitExpr( ctx.expr( 0 ), table, joins ), op, pos, src ); + } else { + throw new UnsupportedOperationException( "Unimplemented expression: " + src ); + } + } + + /** + * Visit the class or interface context to generate the AST node for the + * top level node + * + * @param ctx the parse tree + * + * @return the AST node representing the class or interface + */ + public SQLExpression visitPredicate( PredicateContext ctx, SQLTable table, List joins ) { + var pos = tools.getPosition( ctx ); + var src = tools.getSourceText( ctx ); + + if ( ctx.EQ() != null || ctx.ASSIGN() != null || ctx.IS_() != null ) { + // IS vs IS NOT + SQLBinaryOperator operator = ctx.NOT_() != null ? SQLBinaryOperator.NOTEQUAL : SQLBinaryOperator.EQUAL; + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), operator, pos, src, table, joins ); + } else if ( ctx.BETWEEN_() != null ) { + return new SQLBetweenOperation( visitExpr( ctx.expr( 0 ), table, joins ), visitExpr( ctx.expr( 1 ), table, joins ), + visitExpr( ctx.expr( 2 ), table, joins ), ctx.NOT_() != null, pos, src ); + } else if ( ctx.AND_() != null ) { + // Needs to run AFTER between checks + return binarySimple( ctx.predicate( 0 ), ctx.predicate( 1 ), SQLBinaryOperator.AND, pos, src, table, joins ); + } else if ( ctx.OR_() != null ) { + return binarySimple( ctx.predicate( 0 ), ctx.predicate( 1 ), SQLBinaryOperator.OR, pos, src, table, joins ); } else if ( ctx.IN_() != null ) { SQLExpression expr = visitExpr( ctx.expr( 0 ), table, joins ); if ( ctx.subquery_no_alias() != null ) { @@ -533,18 +586,6 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j escape = visitExpr( ctx.expr( 2 ), table, joins ); } return new SQLBinaryOperation( visitExpr( ctx.expr( 0 ), table, joins ), visitExpr( ctx.expr( 1 ), table, joins ), op, escape, pos, src ); - } else if ( ctx.PIPE2() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.CONCAT, pos, src, table, joins ); - } else if ( ctx.STAR() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MULTIPLY, pos, src, table, joins ); - } else if ( ctx.DIV() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.DIVIDE, pos, src, table, joins ); - } else if ( ctx.MOD() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MODULO, pos, src, table, joins ); - } else if ( ctx.PLUS() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.PLUS, pos, src, table, joins ); - } else if ( ctx.MINUS() != null ) { - return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MINUS, pos, src, table, joins ); } else if ( ctx.LT() != null ) { return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.LESSTHAN, pos, src, table, joins ); } else if ( ctx.LT_EQ() != null ) { @@ -558,20 +599,6 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j } else if ( ctx.OPEN_PAR() != null ) { // Needs to run AFTER function and IN checks return new SQLParenthesis( visitExpr( ctx.expr( 0 ), table, joins ), pos, src ); - } else if ( ctx.case_expr() != null ) { - return visitCase( ctx.case_expr(), table, joins ); - } else if ( ctx.unary_operator() != null ) { - SQLUnaryOperator op; - if ( ctx.unary_operator().BANG() != null ) { - op = SQLUnaryOperator.NOT; - } else if ( ctx.unary_operator().MINUS() != null ) { - op = SQLUnaryOperator.MINUS; - } else if ( ctx.unary_operator().PLUS() != null ) { - op = SQLUnaryOperator.PLUS; - } else { - throw new UnsupportedOperationException( "Unimplemented unary operator: " + ctx.unary_operator().getText() ); - } - return new SQLUnaryOperation( visitExpr( ctx.expr( 0 ), table, joins ), op, pos, src ); } else { throw new UnsupportedOperationException( "Unimplemented expression: " + src ); } @@ -594,8 +621,13 @@ private SQLExpression visitCase( Case_exprContext ctx, SQLTable table, List joins ) { + return new SQLBinaryOperation( visitPredicate( left, table, joins ), visitPredicate( right, table, joins ), op, pos, src ); + } + /** * Visit the class or interface context to generate the AST node for the * top level node diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index 84eb4d97e..9ea1d07c8 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -80,8 +80,9 @@ public class QueryParameter { */ private QueryParameter( IStruct param ) { String sqltype = ( String ) param.getOrDefault( Key.sqltype, "VARCHAR" ); - this.isNullParam = Boolean.TRUE.equals( param.getOrDefault( Key.nulls, false ) ); - this.isListParam = Boolean.TRUE.equals( param.getOrDefault( Key.list, false ) ); + // allow nulls and null + this.isNullParam = BooleanCaster.cast( param.getOrDefault( Key.nulls, param.getOrDefault( Key.nulls2, false ) ) ); + this.isListParam = BooleanCaster.cast( param.getOrDefault( Key.list, false ) ); Object v = param.get( Key.value ); if ( this.isListParam ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 205f63c02..8d93f8a29 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -1,3 +1,4 @@ + /** * [BoxLang] * @@ -68,6 +69,9 @@ public class QoQExecutionService { * @return the AST */ public static SQLNode parseSQL( String sql ) { + if ( sql.indexOf( "bradtest" ) > -1 ) { + // System.out.println( sql ); + } DynamicObject trans = frTransService.startTransaction( "BL QoQ Parse", "" ); SQLParser parser = new SQLParser(); ParsingResult result; @@ -244,25 +248,36 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta // Enforce top/limit for this select. This would be a "top N" clause in the select or a "limit N" clause BEFORE the order by, which // could exist or all selects in a union. - if ( canEarlyLimit && thisSelectLimit > -1 ) { + if ( canEarlyLimit && !select.isDistinct() && thisSelectLimit > -1 ) { intersections = intersections.limit( thisSelectLimit ); } if ( select.hasAggregateResult() || select.getGroupBys() != null ) { - return executeAggregateSelect( QoQExec, target, intersections ); + target = executeAggregateSelect( QoQExec, target, intersections ); + } else { + final Query finalTarget = target; + // No partitioning, just create the final result set + intersections.forEach( intersection -> { + // System.out.println( Arrays.toString( intersection ) ); + Object[] values = new Object[ resultColumns.size() ]; + int colPos = 0; + // Build up row data as native array + for ( Key key : resultColumns.keySet() ) { + values[ colPos++ ] = resultColumns.get( key ).resultColumn.getExpression().evaluate( QoQExec, intersection ); + } + finalTarget.addRow( values ); + } ); } - // No partitioning, just create the final result set - intersections.forEach( intersection -> { - // System.out.println( Arrays.toString( intersection ) ); - Object[] values = new Object[ resultColumns.size() ]; - int colPos = 0; - // Build up row data as native array - for ( Key key : resultColumns.keySet() ) { - values[ colPos++ ] = resultColumns.get( key ).resultColumn.getExpression().evaluate( QoQExec, intersection ); - } - target.addRow( values ); - } ); + // Apply distinct to the final result set + if ( select.isDistinct() ) { + deDupeQuery( target ); + } + + // If we have a limit for this select, apply it here. + if ( thisSelectLimit > -1 ) { + target.truncate( thisSelectLimit ); + } return target; } @@ -296,6 +311,18 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t QoQExec.addPartition( partitionKey, intersection ); } ); + // If there are aggregates in the select, but no group by, and no records were returned, we return a single empty rows + if ( groupBys == null && QoQExec.getPartitions().isEmpty() ) { + Object[] values = new Object[ resultColumns.size() ]; + for ( Key key : resultColumns.keySet() ) { + SQLResultColumn resultColumn = resultColumns.get( key ).resultColumn; + Object value = resultColumn.getExpression().evaluateAggregate( QoQExec, List.of() ); + values[ resultColumn.getOrdinalPosition() - 1 ] = value; + } + target.addRow( values ); + return target; + } + // Make stream parallel if we have a lot of partitions var partitionStream = QoQExec.getPartitions().values().stream(); if ( QoQExec.getPartitions().size() > 50 ) { @@ -316,6 +343,7 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t } target.addRow( values ); } ); + return target; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index 59935c394..d92818f34 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -200,7 +200,17 @@ public void calculateOrderBys() { for ( var orderBy : selectStatement.getOrderBys() ) { SQLExpression expr = orderBy.getExpression(); if ( expr instanceof SQLColumn column ) { - var match = resultColumns.entrySet().stream().filter( rc -> column.getName().equals( rc.getKey() ) ).findFirst(); + var match = resultColumns.entrySet().stream().filter( rc -> { + // Check if the column name matches the name or alias an expression in the result set, regardless of whether it's a column + if ( column.getName().equals( rc.getKey() ) ) { + return true; + } + // Check if the order by name matches the name of a column in the result set + if ( rc.getValue().resultColumn().getExpression() instanceof SQLColumn c ) { + return column.getName().equals( c.getName() ); + } + return false; + } ).findFirst(); if ( match.isPresent() ) { orderByColumns.add( NameAndDirection.of( match.get().getKey(), orderBy.isAscending() ) ); continue; @@ -220,6 +230,11 @@ public void calculateOrderBys() { if ( isUnion ) { throw new DatabaseException( "The order by clause in a union query must reference a column by name that is in the select list or index." ); } + + if ( select.isDistinct() ) { + throw new DatabaseException( "The order by clause in a distinct query must reference a column by name that is in the select list." ); + } + // TODO: Figure out if this exact expression is already in the result set and use that // To do this, we need something like toString() implemented to compare two expressions for equivalence Key newName = Key.of( "__order_by_column_" + additionalCounter++ ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java index a87e2ba13..f3de883d8 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Max.java @@ -48,6 +48,7 @@ public int getMinArgs() { @Override public Object apply( List args, List expressions ) { Object[] input = args.get( 0 ); + Object max = input[ 0 ]; if ( input.length > 1 ) { for ( int i = 1; i < input.length; i++ ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java index 092a51abd..2f727cd6f 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/aggregate/Min.java @@ -48,6 +48,7 @@ public int getMinArgs() { @Override public Object apply( List args, List expressions ) { Object[] input = args.get( 0 ); + Object max = input[ 0 ]; if ( input.length > 1 ) { for ( int i = 1; i < input.length; i++ ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java index 79ac0aa1e..e53042542 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/IsNull.java @@ -35,17 +35,17 @@ public Key getName() { @Override public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { - return QueryColumnType.OBJECT; + return expressions.get( 0 ).getType( QoQExec ); } @Override public int getMinArgs() { - return 1; + return 2; } @Override public Object apply( List args, List expressions ) { - return args.get( 0 ) == null; + return args.get( 0 ) != null ? args.get( 0 ) : args.get( 1 ); } } diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java index 9adedd9bf..b68bda119 100644 --- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java +++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java @@ -505,6 +505,7 @@ public class Key implements Comparable, Serializable { public static final Key notify = Key.of( "notify" ); public static final Key notifyAll = Key.of( "notifyAll" ); public static final Key nulls = Key.of( "null" ); + public static final Key nulls2 = Key.of( "nulls" ); public static final Key number = Key.of( "number" ); public static final Key number1 = Key.of( "number1" ); public static final Key number2 = Key.of( "number2" ); diff --git a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java index 6a36702a3..c44af6b37 100644 --- a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java +++ b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java @@ -68,6 +68,7 @@ public static QueryColumnType fromString( String type ) { case "nclob" : return OBJECT; case "bit" : + case "boolean" : return BIT; case "nchar" : case "char" : @@ -90,6 +91,7 @@ public static QueryColumnType fromString( String type ) { case "tinyint" : case "smallint" : case "integer" : + case "int" : return INTEGER; case "nvarchar" : case "longvarchar" : diff --git a/src/test/java/external/TestBoxTest.java b/src/test/java/external/TestBoxTest.java new file mode 100644 index 000000000..6e37db8e5 --- /dev/null +++ b/src/test/java/external/TestBoxTest.java @@ -0,0 +1,146 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package external; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import ortus.boxlang.runtime.BoxRuntime; +import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; +import ortus.boxlang.runtime.scopes.IScope; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.scopes.VariablesScope; +import ortus.boxlang.runtime.types.Array; +import ortus.boxlang.runtime.types.IStruct; + +public class TestBoxTest { + + static BoxRuntime instance; + IBoxContext context; + IScope variables; + static Key result = new Key( "result" ); + + @BeforeAll + public static void setUp() { + instance = BoxRuntime.getInstance( true ); + } + + @AfterAll + public static void teardown() { + + } + + @BeforeEach + public void setupEach() { + context = new ScriptingRequestBoxContext( instance.getRuntimeContext() ); + variables = context.getScopeNearby( VariablesScope.name ); + } + + @TestFactory + @Disabled + Stream runDynamicTests() { + + // @formatter:off + instance.executeSource( + """ + application name="testbox runner" mappings={"/testbox":expandPath("/src/test/resources/testbox")}; + + result = new testbox.system.TestBox().runRaw( directory='src.test.java.external.specs' ).getMemento(); + + //println( result ); + + // JUnit doesn't support nesting, so we need to flatten all specs to the same level and append all ancestor suite names + // This funcitnn handles an array of suites, which may have come form a bundle or another suite + function mapSuites( array suiteStats ) { + // Process each suite + return suiteStats.reduce( (acc, suite ) =>{ + // Process each spec at the top of the suite + return acc.append( suite.specStats.map( spec => { + // Pull out the data Junit needs + return { + name : spec.name, + skip : spec.skip, + status : spec.status, + failMessage : spec.failMessage, + failStacktrace : spec.failStacktrace, + error : spec.error, + } + } ) + // Recurse into any nested suites + .append( mapSuites( suite.suiteStats), true ) + // Slap the suite name on the front of each spec (which includes nested specs) + .map( s => { + s.name = suite.name & ' - ' & s.name; + return s; + } ), true ); + }, [] ); + } + + // Flatten the results from all bundles + testResults = result.bundleStats.reduce( (acc, bundle) => { + // If the bundle fell flat and didn't even get to execute the specs, pass along the global error as a failure + if( !isSimpleValue( bundle.globalException ) ) { + // If there are global exceptions, we need to add them to the results + acc.append( { + name : bundle.path, + status : 'Error', + failMessage : bundle.globalException.message, + error : bundle.globalException, + } ); + return acc; + } + // Flatten the suites in this bundle + return acc.append( mapSuites( bundle.suiteStats ) + // Slap the bundle name on the front of each spec from all the nested suites + .map( s => { + s.name = bundle.path & ' - ' & s.name; + return s; + } ), true ); + }, [] ); + //println( testResults ); + """, + context ); + // @formatter:on + Array testResults = variables.getAsArray( Key.of( "testResults" ) ); + + return testResults.stream().map( t -> { + IStruct test = ( IStruct ) t; + return DynamicTest.dynamicTest( + test.getAsString( Key._NAME ), + () -> { + String status = test.getAsString( Key.of( "status" ) ); + if ( status.equalsIgnoreCase( "Failed" ) || status.equalsIgnoreCase( "Error" ) ) { + // + "\n" + test.getAsString( Key.of( "failStacktrace" ) ) + fail( test.getAsString( Key.of( "failMessage" ) ), + ( Throwable ) test.get( Key.of( "error" ) ) ); + } + } + ); + } ); + + } +} diff --git a/src/test/java/external/specs/QoQAggregateTest.cfc b/src/test/java/external/specs/QoQAggregateTest.cfc new file mode 100644 index 000000000..57e03e5e0 --- /dev/null +++ b/src/test/java/external/specs/QoQAggregateTest.cfc @@ -0,0 +1,65 @@ +/** +* Copied from test\tickets\LDEV3632.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function beforeAll() { + q = queryNew( + 'type,num', + 'string,int', + [ + ['test',1], + ['test',2], + ['foo',1] + ] + ); + + } + + function run( testResults , testBox ) { + + describe( 'QofQ Aggregation' , function(){ + + it( 'Can can aggregate unflitered rows' , function() { + actual = QueryExecute( + sql = "SELECT sum(num) as sum, count(*) as count, 4 as brad FROM q", + options = { dbtype: 'query' } + ); + expect( actual ).toBeQuery(); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.sum ).toBe( 4 ); + expect( actual.count ).toBe( 3 ); + expect( actual.brad ).toBe( 4 ); + }); + + it( 'Can can aggregate some filtered rows' , function() { + actual = QueryExecute( + sql = "SELECT sum(num) as sum, count(*) as count, 4 as brad FROM q WHERE type='test'", + options = { dbtype: 'query' } + ); + expect( actual ).toBeQuery(); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.sum ).toBe( 3 ); + expect( actual.count ).toBe( 2 ); + expect( actual.brad ).toBe( 4 ); + }); + + it( 'Can can aggregate all flitered rows' , function() { + actual = QueryExecute( + sql = "SELECT sum(num) as sum, count(*) as count, 4 as brad FROM q WHERE 1=0", + options = { dbtype: 'query' } + ); + expect( actual ).toBeQuery(); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.sum ).toBeNull(); + expect( actual.count ).toBe( 0 ); + expect( actual.brad ).toBe( 4 ); + }); + + }); + + } + + +} + diff --git a/src/test/java/external/specs/QoQBasicTest.cfc b/src/test/java/external/specs/QoQBasicTest.cfc new file mode 100644 index 000000000..accbc6159 --- /dev/null +++ b/src/test/java/external/specs/QoQBasicTest.cfc @@ -0,0 +1,641 @@ +/** +* Copied from test\tickets\LDEV3042.cfc in Lucee +* and test\tickets\LDEV4627.cfc +* and test\tickets\LDEV4641.cfc +*/ +component extends="testbox.system.BaseSpec"{ + + function beforeAll() { + employees = queryNew( 'name,age,email,department,isContract,yearsEmployed,sickDaysLeft,hireDate,isActive,empID,favoriteColor', 'varchar,integer,varchar,varchar,boolean,integer,integer,date,boolean,varchar,varchar', [ + [ 'John Doe',28,'John@company.com','Acounting',false,2,4,createDate(2010,1,21),true,'sdf','red' ], + [ 'Jane Doe',22,'Jane@company.com','Acounting',false,0,8,createDate(2011,2,21),true,'hdfg','blue' ], + [ 'Bane Doe',28,'Bane@company.com','Acounting',true,3,2,createDate(2012,3,21),true,'sdsfsff','green' ], + [ 'Tom Smith',25,'Tom@company.com','Acounting',false,6,4,createDate(2013,4,21),false,'HDFG','yellow' ], + [ 'Harry Johnson',38,'Harry@company.com','IT',false,8,6,createDate(2014,5,21),true,'4ge','purple' ], + [ 'Jason Wood',37,'Jason@company.com','IT',false,19,4,createDate(2015,6,21),true,'ShrtDF','Red' ], + [ 'Doris Calhoun',67,'Doris@company.com','IT',true,3,6,createDate(2016,7,21),true,'sgsdg','Blue' ], + [ 'Mary Root',17,'Mary@company.com','IT',false,8,2,createDate(2017,8,21),true,'','Green' ], + [ 'Aurthur Duff',23,'Aurthur@company.com','IT',false,4,0,createDate(2018,9,21),true,nullValue(),'Yellow' ], + [ 'Luis Hake',29,'Luis@company.com','IT',true,9,5,createDate(2019,10,21),true,nullValue(),'Purple' ], + [ 'Gavin Bezos',46,'Gavin@company.com','HR',false,2,5,createDate(2020,11,21),false,nullValue(),'RED' ], + [ 'Nancy Garmon',57,'Nancy@company.com','HR',false,14,9,createDate(2005,12,21),true,nullValue(),'BLUE' ], + [ 'Tom Zuckerburg',27,'Tom@company.com','HR',true,16,10,createDate(2006,1,21),true,nullValue(),'GREEN' ], + [ 'Richard Gates',62,'Richard@company.com','Executive',false,11,1,createDate(2007,2,21),true,nullValue(),'YELLOW' ], + [ 'Amy Merryweather',58,'Amy@company.com','Executive',false,12,2,createDate(2008,3,21),true,nullValue(),'PURPLE' ] + ] ); + } + function run( testResults , testBox ) { + + describe( 'QofQ' , function(){ + + it( 'Can select *' , function() { + actual = QueryExecute( + sql = "SELECT * FROM employees", + options = { dbtype: 'query' } + ); + expect( actual ).toBeQuery(); + expect( actual.recordcount ).toBe( employees.recordcount ); + }); + + it( 'Can select with extra space in multi-word clause' , function() { + actual = QueryExecute( + sql = "SELECT count(1) from employees where empID is null or empID is not null and isActive not like 'test' and isactive not in ('test')", + options = { dbtype: 'query' } + ); + expect( actual ).toBeQuery(); + expect( actual.recordcount ).toBe( 1 ); + }); + + it( 'Can select with functions' , function() { + actual = QueryExecute( + sql = "SELECT upper(name), lower(email), coalesce( null, age ) FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( employees.recordcount ); + }); + + + it( 'Can select with math operations' , function() { + actual = QueryExecute( + sql = "SELECT yearsEmployed/sickDaysLeft as calc1, + yearsEmployed*sickDaysLeft as calc2, + yearsEmployed-sickDaysLeft as calc3, + yearsEmployed+sickDaysLeft as calc4 + from employees + where sickDaysLeft > 0", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 14 ); + }); + + it( 'Can select with functions that are aliased' , function() { + actual = QueryExecute( + sql = "SELECT upper(name) as name, lower(email) email, coalesce( null, age ) as foo FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( employees.recordcount ); + }); + + it( 'Can select with order bys' , function() { + actual = QueryExecute( + sql = "SELECT * from employees ORDER BY department, isActive desc, name, email", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + expect( actual.email[1] ).toBe( 'Bane@company.com' ); + expect( actual.email[15] ).toBe( 'Mary@company.com' ); + }); + + it( 'Can order by alias' , function() { + actual = QueryExecute( + sql = "SELECT department as dept from employees ORDER BY dept", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + expect( actual.dept[1] ).toBe( 'Acounting' ); + expect( actual.dept[15] ).toBe( 'IT' ); + }); + + it( 'Can order by columns not in select' , function() { + actual = QueryExecute( + sql = "SELECT department from employees ORDER BY department, isActive desc, name, email", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + expect( actual.department[1] ).toBe( 'Acounting' ); + expect( actual.department[15] ).toBe( 'IT' ); + }); + + it( 'Can have extra whitespace in group by and order by clauses' , function() { + actual = QueryExecute( + sql = "SELECT department from employees group by department ORDER BY department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.department[1] ).toBe( 'Acounting' ); + expect( actual.department[4] ).toBe( 'IT' ); + }); + + it( 'Can filter on date column' , function() { + actual = QueryExecute( + sql = "SELECT * from employees where hireDate = '2019-10-21 00:00:00.000'", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.name).toBe( 'Luis Hake' ); + }); + + it( 'Can handle isnull' , function() { + actual = QueryExecute( + sql = "SELECT empid, isNull( empID, 'default' ) as empIDNonNull from employees where email in ( 'Doris@company.com','Mary@company.com','Aurthur@company.com' ) order by email", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 3 ); + expect( actual.empid[1] ).toBeNull(); + expect( actual.empidNonNull[1] ).toBe( 'default' ); + expect( actual.empid[2] ).toBe( 'sgsdg' ); + expect( actual.empidNonNull[2] ).toBe( 'sgsdg' ); + expect( actual.empid[3] ).toBe( '' ); + expect( actual.empidNonNull[3] ).toBe( '' ); + }); + + it( 'Can handle isnull with full null support' , function() { + application nullsupport=true action='update'; + actual = QueryExecute( + sql = "SELECT empid, isNull( empID, 'default' ) as empIDNonNull from employees where email in ( 'Doris@company.com','Mary@company.com','Aurthur@company.com' ) order by email", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 3 ); + expect( actual.empid[1] ).toBeNull(); + expect( actual.empidNonNull[1] ).toBe( 'default' ); + expect( actual.empid[2] ).toBe( 'sgsdg' ); + expect( actual.empidNonNull[2] ).toBe( 'sgsdg' ); + expect( actual.empid[3] ).toBe( '' ); + expect( actual.empidNonNull[3] ).toBe( '' ); + + application nullsupport=false action='update'; + }); + + describe( 'Distinct' , function(){ + + it( 'Can select distinct' , function() { + actual = QueryExecute( + sql = "SELECT distinct department FROM employees ORDER BY department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.department[1] ).toBe( 'Acounting' ); + expect( actual.department[2] ).toBe( 'Executive' ); + expect( actual.department[3] ).toBe( 'HR' ); + expect( actual.department[4] ).toBe( 'IT' ); + }); + + it( 'Can select distinct with order by' , function() { + actual = QueryExecute( + sql = "SELECT distinct department FROM employees order by department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + }); + + it( 'Can select distinct with top' , function() { + actual = QueryExecute( + sql = "SELECT top 2 distinct department as foo FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 2 ); + }); + + it( 'Can select distinct with maxrows' , function() { + actual = QueryExecute( + sql = "SELECT distinct department FROM employees order by department", + options = { dbtype: 'query', maxrows: 2 } + ); + expect( actual.recordcount ).toBe( 2 ); + expect( actual.department[1] ).toBe( 'Acounting' ); + expect( actual.department[2] ).toBe( 'Executive' ); + }); + + it( 'Can select distinct with *' , function() { + actual = QueryExecute( + sql = "SELECT distinct * FROM employees", + options = { dbtype: 'query'} + ); + expect( actual.recordcount ).toBe( employees.recordcount ); + }); + + }); + + describe( 'Query Union' , function(){ + + it( 'Can union' , function() { + actual = QueryExecute( + sql = "SELECT * FROM employees + union + SELECT * FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + }); + + it( 'Can union all' , function() { + actual = QueryExecute( + sql = "SELECT * FROM employees + union all + SELECT * FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 30 ); + }); + + it( 'Can union distinct' , function() { + actual = QueryExecute( + sql = "SELECT upper( favoriteColor ) as favoriteColor FROM employees + union distinct + SELECT upper( favoriteColor ) FROM employees + order by favoriteColor", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 5 ); + expect( actual.favoriteColor[1] ).toBeWithCase( 'BLUE' ); + expect( actual.favoriteColor[2] ).toBeWithCase( 'GREEN' ); + expect( actual.favoriteColor[3] ).toBeWithCase( 'PURPLE' ); + expect( actual.favoriteColor[4] ).toBeWithCase( 'RED' ); + expect( actual.favoriteColor[5] ).toBeWithCase( 'YELLOW' ); + }); + + it( 'Can union with top' , function() { + actual = QueryExecute( + sql = "SELECT top 2 * FROM employees + union all + SELECT top 3 * FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 5 ); + }); + + it( 'Can union with maxrows' , function() { + actual = QueryExecute( + sql = "SELECT * FROM employees + union all + SELECT * FROM employees", + options = { dbtype: 'query', maxrows: 2 } + ); + expect( actual.recordcount ).toBe( 2 ); + }); + + it( 'Can union with order' , function() { + actual = QueryExecute( + sql = "SELECT * FROM employees + union + SELECT * FROM employees + order by department, name desc", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + expect( actual.email[1] ).toBe( 'Tom@company.com' ); + expect( actual.email[15] ).toBe( 'Aurthur@company.com' ); + }); + + it( 'Can union with group by' , function() { + actual = QueryExecute( + sql = "SELECT department as thing, count(1) as count, max(age) as age FROM employees + GROUP BY department + union all + SELECT age, count(*), min(sickDaysLeft) FROM employees + GROUP BY age + ORDER BY thing, age", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 18 ); + expect( actual.thing[1] ).toBe( 17 ); + expect( actual.count[1] ).toBe( 1 ); + expect( actual.age[1] ).toBe( 2 ); + expect( actual.thing[18] ).toBe( 'IT' ); + expect( actual.count[18] ).toBe( 6 ); + expect( actual.age[18] ).toBe( 67 ); + }); + + it( 'Can union with literals' , function() { + actual = QueryExecute( + sql = "SELECT TOP 1 'brad' as firstname, 'wood' as lastname FROM employees + union all SELECT TOP 1 'Scott', 'Steinbeck' FROM employees + union all SELECT TOP 1 'Gavin', 'Pickin' FROM employees + union all SELECT TOP 1 'Luis', 'Majano' FROM employees + ", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.firstname[1] ).toBe( 'brad' ); + expect( actual.firstname[4] ).toBe( 'Luis' ); + }); + + }); + + describe( 'Query grouping' , function(){ + + it( 'Can use aggregates with no group by' , function() { + actual = QueryExecute( + sql = "SELECT avg(age) as avgAge, count(1) as totalEmps, max(hireDate) as mostRecentHire, min(sickDaysLeft) as fewestSickDays FROM employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.avgAge ).toBe( 37.6 ); + expect( actual.totalEmps ).toBe( 15 ); + expect( actual.mostRecentHire ).toBe( "{ts '2020-11-21 00:00:00'}" ); + expect( actual.fewestSickDays ).toBe( 0 ); + }); + + it( 'Can use count all and count distinct' , function() { + actual = QueryExecute( + sql = "SELECT count(1) as cNum, + count(*) as cStar, + count('asdf') as cLiteral, + count(name) as cColumn , + count( all department ) as cDeptAll, + count( distinct department) as cDept, + count( distinct empID ) as cEmpID, + count( distinct favoriteColor ) as cfavoriteColor, + count( distinct upper( favoriteColor ) ) as cUpperfavoriteColor + from employees", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.cNum ).toBe( 15 ); + expect( actual.cStar ).toBe( 15 ); + expect( actual.cLiteral ).toBe( 15 ); + expect( actual.cColumn ).toBe( 15 ); + expect( actual.cDeptAll ).toBe( 15 ); + expect( actual.cDept ).toBe( 4 ); + expect( actual.cEmpID ).toBe( 8 ); + expect( actual.cfavoriteColor ).toBe( 15 ); + expect( actual.cUpperfavoriteColor ).toBe( 5 ); + }); + + it( 'Can use aggregates with no group by and where' , function() { + actual = QueryExecute( + sql = "SELECT sum(age) sumAge FROM employees where department = 'IT'", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + expect( actual.sumAge ).toBe( 211 ); + }); + + it( 'Can use group by' , function() { + actual = QueryExecute( + sql = "SELECT department as dept FROM employees GROUP BY department ORDER BY department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.dept[1] ).toBe( 'Acounting' ); + expect( actual.dept[2] ).toBe( 'Executive' ); + expect( actual.dept[3] ).toBe( 'HR' ); + expect( actual.dept[4] ).toBe( 'IT' ); + }); + + it( 'Can use group by with distinct' , function() { + actual = QueryExecute( + sql = "SELECT distinct department as dept FROM employees GROUP BY department ORDER BY department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.dept[1] ).toBe( 'Acounting' ); + expect( actual.dept[2] ).toBe( 'Executive' ); + expect( actual.dept[3] ).toBe( 'HR' ); + expect( actual.dept[4] ).toBe( 'IT' ); + }); + + it( 'Can use group by with aggregates' , function() { + actual = QueryExecute( + sql = "SELECT department as dept, max(hireDate) as mostRecentHire, min(age) as youngestAge, max( email ) FROM employees GROUP BY department order by mostRecentHire desc", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.dept[1] ).toBe( 'HR' ); + expect( actual.mostRecentHire[1] ).toBe( '2020-11-21 00:00:00' ); + expect( actual.dept[4] ).toBe( 'Executive' ); + expect( actual.mostRecentHire[4] ).toBe( '2008-03-21 00:00:00' ); + }); + + it( 'Can use group by with more than one group by' , function() { + actual = QueryExecute( + sql = "SELECT department, isContract, isActive FROM employees GROUP BY department, isContract, isActive ORDER BY department, isContract, isActive", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 9 ); + expect( actual.department[1] ).toBe( 'Acounting' ); + expect( actual.department[5] ).toBe( 'HR' ); + expect( actual.department[9] ).toBe( 'IT' ); + }); + + it( 'Can use group by with having clause' , function() { + actual = QueryExecute( + sql = "SELECT department, max(age) as maxAge from employees GROUP BY department HAVING max(age) > 30 ORDER BY department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 3 ); + expect( actual.department[1] ).toBe( 'Executive' ); + expect( actual.department[2] ).toBe( 'HR' ); + expect( actual.department[3] ).toBe( 'IT' ); + expect( actual.maxAge[1] ).toBe( 62 ); + expect( actual.maxAge[2] ).toBe( 57 ); + expect( actual.maxAge[3] ).toBe( 67 ); + }); + + it( 'Can use group by with having clause and distinct' , function() { + actual = QueryExecute( + sql = "SELECT department, max(age) as maxAge from employees GROUP BY department HAVING max(age) > 30 ORDER BY department", + options = { dbtype: 'query' } + ) + expect( actual.recordcount ).toBe( 3 ); + expect( actual.department[1] ).toBe( 'Executive' ); + expect( actual.department[2] ).toBe( 'HR' ); + expect( actual.department[3] ).toBe( 'IT' ); + expect( actual.maxAge[1] ).toBe( 62 ); + expect( actual.maxAge[2] ).toBe( 57 ); + expect( actual.maxAge[3] ).toBe( 67 ); + }); + + it( 'Can use group by with operations' , function() { + actual = QueryExecute( + sql = "SELECT lower(department) as lowerDept from employees GROUP BY upper(department) HAVING max(age) > 30 ORDER BY lower(department)", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 3 ); + expect( actual.lowerDept[1] ).toBeWithCase( 'executive' ); + expect( actual.lowerDept[2] ).toBeWithCase( 'hr' ); + expect( actual.lowerDept[3] ).toBeWithCase( 'it' ); + }); + + it( 'Can use group by with columns not in select' , function() { + actual = QueryExecute( + sql = "SELECT 'test' as val from employees GROUP BY department, age, lower(email) ORDER BY upper(department)", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 15 ); + expect( actual.val[1] ).toBe( 'test' ); + expect( actual.val[15] ).toBe( 'test' ); + }); + + it( 'Can order by aggregate columns' , function() { + actual = QueryExecute( + sql = "SELECT department, max(age) as maxAge from employees GROUP BY department HAVING max(age) > 30 ORDER BY max(age)", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 3 ); + expect( actual.department[1] ).toBe( 'HR' ); + expect( actual.department[2] ).toBe( 'Executive' ); + expect( actual.department[3] ).toBe( 'IT' ); + expect( actual.maxAge[1] ).toBe( 57 ); + expect( actual.maxAge[2] ).toBe( 62 ); + expect( actual.maxAge[3] ).toBe( 67 ); + }); + + it( 'Can reference more than one column in aggregate function' , function() { + actual = QueryExecute( + sql = "SELECT department, sum(yearsEmployed * sickDaysLeft) as calc from employees GROUP BY department order by department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.calc[1] ).toBe( 38 ); + expect( actual.calc[2] ).toBe( 35 ); + expect( actual.calc[3] ).toBe( 296 ); + expect( actual.calc[4] ).toBe( 203 ); + }); + + it( 'Can wrap aggregate function in scalar function' , function() { + actual = QueryExecute( + sql = "SELECT floor(sum(yearsEmployed * sickDaysLeft)) as calc from employees group by department order by department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.calc[1] ).toBe( 38 ); + expect( actual.calc[2] ).toBe( 35 ); + expect( actual.calc[3] ).toBe( 296 ); + expect( actual.calc[4] ).toBe( 203 ); + }); + + it( 'Can nest scalar functions inside of aggregates inside of scalar functions and use more than aggregate in a single operation' , function() { + actual = QueryExecute( + sql = "SELECT department, max( yearsEmployed ) as max, count(1) as count, ceiling( max( floor( yearsEmployed ) )+count(1) ) as calc from employees group by department order by department", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 4 ); + expect( actual.calc[1] ).toBe( 10 ); + expect( actual.calc[2] ).toBe( 14 ); + expect( actual.calc[3] ).toBe( 19 ); + expect( actual.calc[4] ).toBe( 25 ); + }); + + it( 'Aggregate select with no group by against empty query returns 1 row of empty strings' , function() { + var qry = queryNew( 'col', 'varchar' ); + actual = QueryExecute( + sql = "SELECT 'const' as const, count(1) as count, avg(col) as avg, min(col) as min, max(col) as max, isNull( max(col), 'test' ) as max2 from qry", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + // Count always returns a number + expect( actual.count[1] ).toBe( 0 ); + // Other aggregates return empty + expect( actual.avg[1] ).toBeNull(); + expect( actual.min[1] ).toBeNull(); + expect( actual.max[1] ).toBeNull(); + // Constant value is just passed through + expect( actual.const[1] ).toBe( 'const' ); + // Scalar function still processes + expect( actual.max2[1] ).toBe( 'test' ); + }); + + it( 'Aggregate select with no group by and a where clause against empty query returns 1 row of empty strings' , function() { + var qry = queryNew( 'col', 'varchar' ); + actual = QueryExecute( + sql = "SELECT 'const' as const, count(1) as count, avg(col) as avg, min(col) as min, max(col) as max, isNull( max(col), 'test' ) as max2 from qry where col = ''", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + // Count always returns a number + expect( actual.count[1] ).toBe( 0 ); + // Other aggregates return empty + expect( actual.avg[1] ).toBeNull(); + expect( actual.min[1] ).toBeNull(); + expect( actual.max[1] ).toBeNull(); + // Constant value is just passed through + expect( actual.const[1] ).toBe( 'const' ); + // Scalar function still processes + expect( actual.max2[1] ).toBe( 'test' ); + }); + + it( 'Aggregate select with group by against empty query returns 0 rows' , function() { + var qry = queryNew( 'col', 'varchar' ); + actual = QueryExecute( + sql = "SELECT count(1) as count from qry group by col", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 0 ); + }); + + it( 'Aggregates ignore null values' , function() { + var qry = queryNew( 'col,col2', 'integer,integer', [ + [ 0, nullValue() ], + [ nullValue(), nullValue() ], + [ 100, nullValue() ], + ] ); + actual = QueryExecute( + sql = "SELECT sum(col) as sum, avg(col) as avg, min(col) as min, max(col) as max, sum(col2) as sum2, avg(col2) as avg2, min(col2) as min2, max(col2) as max2 from qry", + options = { dbtype: 'query' } + ); + expect( actual.recordcount ).toBe( 1 ); + + // When some column values are null, aggregates should ignore the null values + expect( actual.sum[1] ).toBe( 100 ); + expect( actual.avg[1] ).toBe( 50 ); + expect( actual.min[1] ).toBe( 0 ); + expect( actual.max[1] ).toBe( 100 ); + + // When all column values are null, aggregates should return nothing + expect( actual.sum2[1] ).toBeNull(); + expect( actual.avg2[1] ).toBeNull(); + expect( actual.min2[1] ).toBeNull(); + expect( actual.max2[1] ).toBeNull(); + }); + + }); + + + it( 'can handle column names of different case' , ()=>{ + qry = queryNew( 'col', 'varchar', [['foo'],['bar']] ); + var actual = QueryExecute( + sql = " + SELECT distinct col + FROM qry + where COL = 'foo'", + options = { dbtype: 'query' } + ); + + expect( actual.recordcount ).toBe( 1 ); + + }); + + describe( 'check column case with qoq' , () =>{ + + it( 'column case is preserved (mixed)' , ()=>{ + var qMaster = queryNew( 'ID, THours', 'numeric,numeric', [[1000, 6],[1000, 5]] ); + var actual = QueryExecute( + sql = " + SELECT ID, sum(THours) AS THours + FROM qMaster + GROUP BY id", + options = { dbtype: 'query' } + ); + + expect( actual.recordcount ).toBe( 1 ); + expect( actual.thours ).toBe( 11 ); + expect( listFirst( actual.columnList ,"," ) ).toBeWithCase( 'ID' ); + + }); + + it( 'column case is preserved (same)' , ()=>{ + var qMaster = queryNew( 'ID, THours', 'numeric,numeric', [[1000, 6],[1000, 5]] ); + var actual = QueryExecute( + sql = " + SELECT ID, sum(THours) AS THours + FROM qMaster + GROUP BY ID", + options = { dbtype: 'query' } + ); + + expect( actual.recordcount ).toBe( 1 ); + expect( actual.thours ).toBe( 11 ); + expect( listFirst( actual.columnList ,"," ) ).toBeWithCase( 'ID' ); + + }); + + + }); + + }); + + } + +} + diff --git a/src/test/java/external/specs/QoQCountStructureTest.cfc b/src/test/java/external/specs/QoQCountStructureTest.cfc new file mode 100644 index 000000000..91047b3f1 --- /dev/null +++ b/src/test/java/external/specs/QoQCountStructureTest.cfc @@ -0,0 +1,92 @@ +/** +* Copied from test\tickets\_LDEV0691.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults , testBox ) { + describe( title="Test suite for LDEV-691", body=function() { + it( title='Checking QuQ Count with structure data',body=function( currentSpec ) { + var obj = { a=1, b=2, c=3 }; + var q_obj = QueryNew( "name,data,age", "varchar,varchar,Integer", [ + [ "Susi", obj, 24 ], + [ "Urs" , "switz", 55 ], + [ "Fred", "India", 45 ], + [ "Jim" , "USA", 55 ] + ]); + + try { + var qoq = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + select count(*) as r, age + from q_obj + group by age" + ); + var result = qoq.recordCount + } catch ( any e){ + var result = e.message; + } + + expect( result ).toBe( 3 ); + }); + + it( title='Checking QuQ Count with array data',body=function( currentSpec ) { + var obj = [1,2,3,4]; + var q_obj = QueryNew( "name,data,age", "varchar,varchar,Integer", [ + [ "Susi", obj, 24 ], + [ "Urs" , "switz", 55 ], + [ "Fred", "India", 45 ], + [ "Jim" , "USA", 55 ] + ]); + + try { + var qoq = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + select count(*) as r, age + from q_obj + group by age" + ); + var result = qoq.recordCount + } catch ( any e){ + var result = e.message; + } + + expect( result ).toBe( 3 ); + }); + + it( title='Checking QuQ Count with query data',body=function( currentSpec ) { + var obj = queryNew("test1, test2"); + var q_obj = QueryNew( "name,data,age", "varchar,varchar,Integer", [ + [ "Susi", obj, 24 ], + [ "Urs" , "switz", 55 ], + [ "Fred", "India", 45 ], + [ "Jim" , "USA", 55 ] + ]); + + try { + var qoq = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + select count(*) as r, age + from q_obj + group by age" + ); + var result = qoq.recordCount + } catch ( any e){ + var result = e.message; + } + + expect( result ).toBe( 3 ); + }); + }); + } + +} + diff --git a/src/test/java/external/specs/QoQDistinctEdgeCaseTest.cfc b/src/test/java/external/specs/QoQDistinctEdgeCaseTest.cfc new file mode 100644 index 000000000..c56b785ca --- /dev/null +++ b/src/test/java/external/specs/QoQDistinctEdgeCaseTest.cfc @@ -0,0 +1,34 @@ +/** +* Copied from test\tickets\LDEV3830.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + + function run( testResults, textbox ) { + + describe("testcase for LDEV-3830", function(){ + + it(title="QoQ distinct rows with dupes in same original result", body=function( currentSpec ){ + var a = queryNew("a","varchar"); + var b = queryNew("a","varchar", [['1'],['2'],['2']]); + + var actual = QueryExecute( + sql = " + select a + from a + union select a + from b", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( actual.recordCount ).toBe( 2 ); + + }); + + }); + + } + +} + diff --git a/src/test/java/external/specs/QoQDistinctSortTest.cfc b/src/test/java/external/specs/QoQDistinctSortTest.cfc new file mode 100644 index 000000000..651fdcb47 --- /dev/null +++ b/src/test/java/external/specs/QoQDistinctSortTest.cfc @@ -0,0 +1,27 @@ +/** +* Copied from test\tickets\LDEV3822.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + + describe("testcase for LDEV-3822", function(){ + + it(title="throws exception when using select distinct and ordering by a column not in the select list", body=function( currentSpec ){ + var employees = queryNew( 'name,age', 'varchar,integer', + [ [ 'John Doe',28 ], + [ 'Jane Doe',28 ], + [ 'Bane Doe',28 ]] ); + + expect( ()=>QueryExecute( + sql = "SELECT DISTINCT age from employees ORDER BY age, name", + queryoptions = { dbtype: 'query' } + ) ).toThrow(); + + }); + + }); + } + +} + diff --git a/src/test/java/external/specs/QoQDupeSelectTest.cfc b/src/test/java/external/specs/QoQDupeSelectTest.cfc new file mode 100644 index 000000000..5c46bea4b --- /dev/null +++ b/src/test/java/external/specs/QoQDupeSelectTest.cfc @@ -0,0 +1,49 @@ +/** +* Copied from test\tickets\LDEV3801.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + + describe("testcase for LDEV-3801", function(){ + + it(title="can select the same column twice without error in QoQ ORDER BY", body=function( currentSpec ){ + var employees = queryNew( 'name,age,email', 'varchar,integer,varchar',[ + ['Brad',20,'brad@test.com'], + ['Luis',10,'luis@test.com'] + ]); + + var actual = QueryExecute( + sql = "SELECT name, age, name from employees ORDER BY age, email", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( actual.recordCount ).toBe( 2 ); + expect( queryColumnData( actual, 'age' )[2] ).toBe( 20 ); + + }); + + it(title="Select the same column twice without error in QoQ ORDER BY again", body=function( currentSpec ){ + var employees = queryNew( 'name,age,email', 'varchar,integer,varchar',[ + ['Brad',20,'brad@test.com'], + ['Luis',10,'luis@test.com'] + ]); + + var actual = QueryExecute( + sql = "SELECT DISTINCT age, name, age, email from employees ORDER BY age, email", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( actual.recordCount ).toBe( 2 ); + expect( queryColumnData( actual, 'age' )[2] ).toBe( 20 ); + + }); + + }); + + } + +} + diff --git a/src/test/java/external/specs/QoQLists2Test.cfc_ b/src/test/java/external/specs/QoQLists2Test.cfc_ new file mode 100644 index 000000000..102ca138e --- /dev/null +++ b/src/test/java/external/specs/QoQLists2Test.cfc_ @@ -0,0 +1,281 @@ +/** +* Copied from test\tickets\LDEV0224.cfc in Lucee +* Disabled since list params don't seem to work yet +*/ +component extends="testbox.system.BaseSpec"{ + + + function beforeAll() { + variables.interestingNumbersAsAList = '3,4'; + variables.interestingStringsAsAList = "a,c,e"; + variables.interestingStringsAsAQuotedList = "'a','c','e'"; + + variables.queryWithDataIn = QueryNew('id,value', 'integer,varchar',[[1,'a'],[2,'b'],[3,'c'],[4,'d'],[5,'e']]); + } + + function run( testResults , testBox ) { + + describe( 'selecting 2 rows from QoQ' , function() { + + describe( 'is possible using a hard coded list' , function() { + + it( 'of numerics' , function( currentSpec ) { + + var actual = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( "&interestingNumbersAsAList&" ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + it( 'of strings' , function( currentSpec ) { + + var actual = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( "&interestingStringsAsAQuotedList&" ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAQuotedList , ',' ) ); + + }); + + + }); + + describe( 'using param list=true' , function() { + + describe( 'with query{} ( cfquery )' , function() { + + describe( 'returns expected rows' , function() { + + it( 'when using numeric params' , function( currentSpec ) { + + query + name = 'actual' + dbtype = 'query' { + + WriteOutput( " + SELECT + id, + value, + 'bradtest' + FROM queryWithDataIn + WHERE id IN ( " + ); + + queryparam + value = interestingNumbersAsAList + sqltype = 'integer' + list = true; + + WriteOutput( " )" ); + } + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + it( 'when using numeric params and a custom separator' , function( currentSpec ) { + + query + name = 'actual' + dbtype = 'query' { + + WriteOutput( " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( " + ); + + queryparam + value = Replace( interestingNumbersAsAList , ',' , '|' ) + sqltype = 'integer' + list = true + separator = '|'; + + WriteOutput( " )" ); + } + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + it( 'when using string params' , function( currentSpec ) { + + query + name = 'actual' + dbtype = 'query' { + + WriteOutput( " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( " + ); + + queryparam + value = interestingStringsAsAList + sqltype = 'varchar' + list = true; + + WriteOutput( " )" ); + } + + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + + }); + }); + }); + + describe( 'with QueryExecute' , function() { + + describe( 'returns expected rows' , function() { + + it( 'when using an array of numeric params' , function( currentSpec ) { + + var actual = QueryExecute( + params = [ + { name: 'needle' , value: interestingNumbersAsAList , sqltype: 'numeric' , list = true } + ], + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + + it( 'when using a struct of numeric params' , function( currentSpec ) { + + var actual = QueryExecute( + params = { + needle: { value: interestingNumbersAsAList , sqltype: 'numeric' , list: true } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + it( 'when using an array of string params' , function( currentSpec ) { + + var actual = QueryExecute( + params = [ + { name: 'needle' , value: interestingStringsAsAList , sqltype: 'varchar' , list = true } + ], + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( :needle ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + + }); + + + it( 'when using a struct of string params' , function( currentSpec ) { + + var actual = QueryExecute( + params = { + needle: { value: interestingStringsAsAList , sqltype: 'varchar' , list: true } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( :needle ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + + }); + + + it( 'when using numeric params and a custom separator' , function( currentSpec ) { + + var actual = QueryExecute( + params = { + needle: { value: Replace( interestingNumbersAsAList , ',' , '|' ) , sqltype: 'numeric' , list: true , separator: '|' } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + + }); + + }); + + }); + + }); + + }); + + + } + + + +} + diff --git a/src/test/java/external/specs/QoQListsTest.cfc_ b/src/test/java/external/specs/QoQListsTest.cfc_ new file mode 100644 index 000000000..27a6e5bdb --- /dev/null +++ b/src/test/java/external/specs/QoQListsTest.cfc_ @@ -0,0 +1,266 @@ +/** +* Copied from test\tickets\LDEV0224_1.cfc in Lucee +* Disabled since list params don't seem to work yet +*/ +component extends="testbox.system.BaseSpec"{ + + function beforeAll() { + variables.interestingNumbersAsAList = '3,4'; + variables.interestingStringsAsAList = "a,c,e"; + variables.interestingStringsAsAQuotedList = "'a','c','e'"; + + variables.queryWithDataIn = Query( + id: [ 1 , 2 , 3 , 4 , 5 ], + value: [ 'a' , 'b' , 'c' , 'd' , 'e' ] + ); + } + + function run( testResults , testBox ) { + describe( title='selecting 2 rows from QoQ' , body=function() { + describe( title='is possible using a hard coded list' , body=function() { + it( title='of numerics' , body=function( currentSpec ) { + var actual = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( "&interestingNumbersAsAList&" ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='of strings' , body=function( currentSpec ) { + var actual = QueryExecute( + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( "&interestingStringsAsAQuotedList&" ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAQuotedList , ',' ) ); + }); + }); + + describe( title='using param list=true' , body=function() { + describe( title='with new Query()' , body=function() { + beforeEach( function( currentSpec ) { + q = new Query( + dbtype = 'query', + queryWithDataIn = variables.queryWithDataIn + ); + }); + + it( title='when using numeric params' , body=function( currentSpec ) { + q.addParam( name: 'needle' , value: interestingNumbersAsAList , sqltype: 'numeric' , list: true ); + var actual = q.execute( sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " ).getResult(); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using numeric params and a custom separator' , body=function( currentSpec ) { + q.addParam( name: 'needle' , value: Replace( interestingNumbersAsAList , ',' , '|' ) , sqltype: 'numeric' , list: true , separator: '|' ); + var actual = q.execute( sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " ).getResult(); + + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using string params' , body=function( currentSpec ) { + q.addParam( name: 'needle' , value: interestingStringsAsAList , sqltype: 'varchar' , list: true ); + var actual = q.execute( sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( :needle ) + " ).getResult(); + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + }); + }); + + describe( title='with query{} ( cfquery )' , body=function() { + it( title='when using numeric params' , body=function( currentSpec ) { + query + name = 'actual' + dbtype = 'query' { + WriteOutput( " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( " + ); + queryparam + value = interestingNumbersAsAList + sqltype = 'integer' + list = true; + WriteOutput( " )" ); + } + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using numeric params and a custom separator' , body=function( currentSpec ) { + query + name = 'actual' + dbtype = 'query' { + WriteOutput( " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( " + ); + queryparam + value = Replace( interestingNumbersAsAList , ',' , '|' ) + sqltype = 'integer' + list = true + separator = '|'; + WriteOutput( " )" ); + } + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using string params' , body=function( currentSpec ) { + query + name = 'actual' + dbtype = 'query' { + WriteOutput( " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( " + ); + queryparam + value = interestingStringsAsAList + sqltype = 'varchar' + list = true; + WriteOutput( " )" ); + } + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + }); + + }); + + describe( title='with QueryExecute' , body=function() { + it( title='when using an array of numeric params' , body=function( currentSpec ) { + var actual = QueryExecute( + params = [ + { name: 'needle' , value: interestingNumbersAsAList , sqltype: 'numeric' , list = true } + ], + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using a struct of numeric params' , body=function( currentSpec ) { + var actual = QueryExecute( + params = { + needle: { value: interestingNumbersAsAList , sqltype: 'numeric' , list: true } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + + it( title='when using an array of string params' , body=function( currentSpec ) { + var actual = QueryExecute( + params = [ + { name: 'needle' , value: interestingStringsAsAList , sqltype: 'varchar' , list = true } + ], + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( :needle ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + }); + + it( title='when using a struct of string params' , body=function( currentSpec ) { + var actual = QueryExecute( + params = { + needle: { value: interestingStringsAsAList , sqltype: 'varchar' , list: true } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE value IN ( :needle ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); + }); + + it( title='when using numeric params and a custom separator' , body=function( currentSpec ) { + var actual = QueryExecute( + params = { + needle: { value: Replace( interestingNumbersAsAList , ',' , '|' ) , sqltype: 'numeric' , list: true , separator: '|' } + }, + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id IN ( :needle ) + " + ); + expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); + }); + }); + }); + }); + } +} + diff --git a/src/test/java/external/specs/QoQNullParamTest.cfc b/src/test/java/external/specs/QoQNullParamTest.cfc new file mode 100644 index 000000000..bde586047 --- /dev/null +++ b/src/test/java/external/specs/QoQNullParamTest.cfc @@ -0,0 +1,148 @@ +/** +* Copied from test\tickets\LDEV0364.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + + function beforeAll() { + variables.queryWithDataIn = Querynew( 'id', 'integer', [[ 1 ]] ); + } + + function run( testResults , testBox ) { + + describe( 'QueryExecute' , function(){ + + it( 'returns NULL when fed in through array parameter with nulls=true' , function() { + + /* + I have no idea *why* this is this way, but in the source code I spotted this. + It turns out that there is an ability to cast to null but the parameter attribute + is "nulls" instead of "null", go figure! + */ + + actual = QueryExecute( + options = { + dbtype: 'query' + }, + params = [ + { type: 'integer' , value: 1 , null: true } + ], + sql = " + SELECT + COALESCE( ? , 'isnull' ) AS value, + COALESCE( NULL , 'isnull' ) AS control + FROM queryWithDataIn + " + ); + + expect( actual.control[1] ).toBe( 'isnull' ); + expect( actual.value[1] ).toBe( 'isnull' ); + + }); + + it( 'returns NULL when fed in through array parameter with null=true' , function() { + + actual = QueryExecute( + options = { + dbtype: 'query' + }, + params = [ + { type: 'integer' , value: 1 , null: true } + ], + sql = " + SELECT + COALESCE( ? , 'isnull' ) AS value, + COALESCE( NULL , 'isnull' ) AS control + FROM queryWithDataIn + " + ); + + expect( actual.control[1] ).toBe( 'isnull' ); + expect( actual.value[1] ).toBe( 'isnull' ); + + }); + + it( 'returns NULL when fed in through array named parameter with null=true' , function() { + + actual = QueryExecute( + options = { + dbtype: 'query' + }, + params = [ + { name: 'input' , type: 'integer' , value: 1 , null: true } + ], + sql = " + SELECT + COALESCE( :input , 'isnull' ) AS value, + COALESCE( NULL , 'isnull' ) AS control + FROM queryWithDataIn + " + ); + + expect( actual.control[1] ).toBe( 'isnull' ); + expect( actual.value[1] ).toBe( 'isnull' ); + + }); + + it( 'returns NULL when fed in through struct parameter with null=true' , function() { + + actual = QueryExecute( + options = { + dbtype: 'query' + }, + params = { + 'input': { type: 'integer' , value: 42 , nulls: true } + }, + sql = " + SELECT + COALESCE( :input , 'isnull-value' ) AS value, + COALESCE( NULL , 'isnull-control' ) AS control + FROM queryWithDataIn + " + ); + + expect( actual.control[1] ).toBe( 'isnull-control' ); + expect( actual.value[1] ).toBe( 'isnull-value' ); + + + }); + + }); + + describe( 'cfquery in script' , function(){ + + it( 'returns NULL when fed in through parameter with null=true' , function() { + + query + name = 'actual' + dbtype = 'query' { + + WriteOutput( " + + SELECT + COALESCE( + " ); + + queryparam + value = 1 + sqltype = 'integer' + null = true; + + WriteOutput( " , 'isnull' ) AS value, + COALESCE( NULL , 'isnull' ) AS control + FROM queryWithDataIn + " ); + } + + expect( actual.control[1] ).toBe( 'isnull' ); + expect( actual.value[1] ).toBe( 'isnull' ); + + }); + + }); + + } + + +} + diff --git a/src/test/java/external/specs/QoQOrderByEdgeCaseTest.cfc b/src/test/java/external/specs/QoQOrderByEdgeCaseTest.cfc new file mode 100644 index 000000000..c635c84b0 --- /dev/null +++ b/src/test/java/external/specs/QoQOrderByEdgeCaseTest.cfc @@ -0,0 +1,81 @@ +/** +* Copied from test\tickets\LDEV3823.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + + + describe("testcase for LDEV-3823", function(){ + + it(title="Select the same column twice without error in QoQ ORDER BY using an ordinal position ORDER BY", body=function( currentSpec ){ + var employees = queryNew( 'name,age,email', 'varchar,integer,varchar',[ + ['Brad',20,'brad@test.com'], + ['Luis',10,'luis@test.com'] + ]); + + var actual = QueryExecute( + sql = "SELECT DISTINCT age, email, age from employees ORDER BY 1", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( actual.recordCount ).toBe( 2 ); + expect( queryColumnData( actual, 'age' )[2] ).toBe( 20 ); + + }); + + it(title="QoQ ORDER BY column named with actual number", body=function( currentSpec ){ + var employees = queryNew( 'name,45', 'varchar,integer',[ + ['Brad',20], + ['Luis',10] + ]); + + // make sure we don't confuse the Literal 45 with the actual column named "45" + var actual = QueryExecute( + sql = "SELECT name, 45, [45] from employees ORDER BY [45]", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( queryColumnData( actual, '45' )[2] ).toBe( 20 ); + + }); + + it(title="QoQ ORDER BY column named with actual boolean", body=function( currentSpec ){ + var employees = queryNew( 'name,true,false', 'varchar,varchar,varchar',[ + ['Brad','yeah','nah1'], + ['Luis','yeah','nah2'] + ]); + + // make sure we don't confuse the Literal true/false with the actual column named "true"/"false" + var actual = QueryExecute( + sql = "SELECT name, true, false, [true], [false] from employees ORDER BY [true], [false] desc", + options = { dbtype: 'query' } + ); + + expect( actual ).toBeQuery(); + expect( queryColumnData( actual, 'false' )[1] ).toBe( 'nah2' ); + expect( queryColumnData( actual, 'false' )[2] ).toBe( 'nah1' ); + + }); + + it(title="QoQ ORDER BY using an ordinal position ORDER BY out of range", body=function( currentSpec ){ + var employees = queryNew( 'name,age,email', 'varchar,integer,varchar',[ + ['Brad',20,'brad@test.com'], + ['Luis',10,'luis@test.com'] + ]); + + expect( ()=>QueryExecute( + sql = "SELECT age from employees ORDER BY 99", + options = { dbtype: 'query' } + ) ).toThrow(); + + }); + + }); + + } + +} + diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 1f1dfac8a..850d6648d 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -64,7 +64,7 @@ public void testMetadataVisitor() { """ select foo, bar b, bum as b2, * from mytable t - where true + where true = true order by t.baz limit 5 """ @@ -129,6 +129,24 @@ public void testRunQoQUnion() { context ); } + @Test + public void testRunQoQUnion2() { + instance.executeSource( + """ + qryDept = queryNew( "name,code", "varchar,integer", [["IT",404],["Exec",200],["Janitor",200]] ) + q = queryExecute( " + select name as col from qryDept + union select name from qryDept + order by col desc + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + } + @Test public void testRunQoQUnionDistinct() { instance.executeSource( @@ -493,4 +511,74 @@ public void testInputCaseNoElse() { context ); } + @Test + public void testDistinct() { + instance.executeSource( + """ + qryEmployees = queryNew( + "name,age,dept,supervisor", + "varchar,integer,varchar,varchar", + [ + ["luis",43,"Exec","luis"], + ["brad",44,"IT","luis"], + ["jacob",35,"IT","luis"], + ["Jon",45,"HR","luis"] + ] + ) + + q = queryExecute( " + select distinct dept as d + from qryEmployees + order by dept + ", + [], + { dbType : "query" } + ); + println( q ) + """, + context ); + assertThat( variables.getAsQuery( q ).size() ).isEqualTo( 3 ); + } + + @Test + public void testNullAggregate() { + instance.executeSource( + """ + + qry = queryNew( 'col,col2', 'integer,integer', [ + [ 0, nullValue() ], + [ nullValue(), nullValue() ], + [ 100, nullValue() ], + ] ); + actual = QueryExecute( + sql = "SELECT sum(col) as sum, avg(col) as avg, min(col) as min, max(col) as max, sum(col2) as sum2, avg(col2) as avg2, min(col2) as min2, max(col2) as max2 + from qry", + options = { dbtype: 'query' } + ); + println( actual ) + """, + context ); + } + + @Test + public void testsdf() { + instance.executeSource( + """ + a = queryNew("a","varchar"); + b = queryNew("a","varchar", [['1'],['2'],['2']]); + + actual = QueryExecute( + sql = " + select a + from a + union select a + from b", + options = { dbtype: 'query' } + ); + + println( actual ) + """, + context ); + } + } diff --git a/src/test/resources/box.json b/src/test/resources/box.json new file mode 100644 index 000000000..a067ebcca --- /dev/null +++ b/src/test/resources/box.json @@ -0,0 +1,8 @@ +{ + "dependencies":{ + "testbox":"be" + }, + "installPaths":{ + "testbox":"testbox/" + } +} From 20555f5a5a53e1267af186af3b94cbe180a0529b Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Fri, 20 Dec 2024 12:17:01 -0600 Subject: [PATCH 042/161] java docs --- .../ortus/boxlang/runtime/types/Struct.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/Struct.java b/src/main/java/ortus/boxlang/runtime/types/Struct.java index 01c5155f3..c95a795f8 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Struct.java +++ b/src/main/java/ortus/boxlang/runtime/types/Struct.java @@ -59,15 +59,15 @@ /** * This type provides the core map class for Boxlang. Structs are highly versatile and are used for organizing and managing related data. * - * Types of Structs in BoxLang: - * - * * Basic Structs: These are the basic structures where each key is associated with a single value. Keys are case-insensitive and can be strings or symbols. - * * Nested Structs: Structs can contain other structs as values, allowing for a hierarchical organization of data. - * * Case-Sensitive Structs: By default, BoxLang structs are case-insensitive. However, you can create case-sensitive structs if needed. - * * Ordered Structs: This implementation of a Struct maintains keys in the order they were added. - * * Sorted Structs: This implementation of a Struct maintains keys in specified sorted order. - * + * Types of Structs in BoxLang: DEFAULT, CASE_SENSITIVE, LINKED, LINKED_CASE_SENSITIVE, SORTED, WEAK, SOFT * + * - DEFAULT: These are the basic structures where each key is associated with a single value. Keys are case-insensitive and can be strings or symbols. + * - Nested Structs: Structs can contain other structs as values, allowing for a hierarchical organization of data. + * - CASE_SENSITIVE: By default, BoxLang structs are case-insensitive. However, you can create case-sensitive structs if needed. + * - LINKED, LINKED_CASE_SENSITIVE (Ordered Structs): This implementation of a Struct maintains keys in the order they were added. + * - SORTED: This implementation of a Struct maintains keys in specified sorted order. + * - WEAK: This implementation of a Struct uses weak references for keys. + * - SOFT: This implementation of a Struct uses a default struct with values wrapped in a SoftReference. */ public class Struct implements IStruct, IListenable, Serializable { @@ -149,9 +149,9 @@ public class Struct implements IStruct, IListenable, Serializable { /** * Constructor * - * @param type The type of struct to create: DEFAULT, LINKED, SORTED + * @param type The type of struct to create: DEFAULT, CASE_SENSITIVE, LINKED, LINKED_CASE_SENSITIVE, SORTED, WEAK, SOFT * - * @throws BoxRuntimeException If an invalid type is specified: DEFAULT, LINKED, SORTED + * @throws BoxRuntimeException If an invalid type is specified: DEFAULT, CASE_SENSITIVE, LINKED, LINKED_CASE_SENSITIVE, SORTED, WEAK, SOFT */ public Struct( TYPES type ) { this.type = type; @@ -159,9 +159,9 @@ public Struct( TYPES type ) { // Initialize the wrapped map this.wrapped = switch ( type ) { case DEFAULT, CASE_SENSITIVE, SOFT -> new ConcurrentHashMap<>( INITIAL_CAPACITY ); - case LINKED, LINKED_CASE_SENSITIVE -> Collections.synchronizedMap( new LinkedHashMap<>( INITIAL_CAPACITY ) ); + case LINKED, LINKED_CASE_SENSITIVE -> Collections.synchronizedMap( LinkedHashMap.newLinkedHashMap( INITIAL_CAPACITY ) ); case SORTED -> new ConcurrentSkipListMap<>(); - case WEAK -> new WeakHashMap<>( INITIAL_CAPACITY ); + case WEAK -> WeakHashMap.newWeakHashMap( INITIAL_CAPACITY ); default -> throw new BoxRuntimeException( "Invalid struct type [" + type.name() + "]" ); }; } From 3626e05fc721e831356ef90751c2a8a98f3fd07c Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Fri, 20 Dec 2024 13:17:00 -0600 Subject: [PATCH 043/161] BL-875 #resolve rework onSessionEnd to make sure the application exists when calling the listeners --- .../boxlang/runtime/application/Session.java | 70 +++++++++++-------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/application/Session.java b/src/main/java/ortus/boxlang/runtime/application/Session.java index d1ff3afec..8ded0fca1 100644 --- a/src/main/java/ortus/boxlang/runtime/application/Session.java +++ b/src/main/java/ortus/boxlang/runtime/application/Session.java @@ -20,12 +20,12 @@ import java.io.Serializable; import ortus.boxlang.runtime.BoxRuntime; -import ortus.boxlang.runtime.context.ApplicationBoxContext; import ortus.boxlang.runtime.context.IBoxContext; -import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; import ortus.boxlang.runtime.events.BoxEvent; +import ortus.boxlang.runtime.scopes.ApplicationScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.SessionScope; +import ortus.boxlang.runtime.services.ApplicationService; import ortus.boxlang.runtime.types.DateTime; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Struct; @@ -206,37 +206,47 @@ public String getCacheKey() { * @param listener The listener that is shutting down the session */ public void shutdown( BaseApplicationListener listener ) { - // Announce it's destruction to the runtime first - BoxRuntime.getInstance() - .getInterceptorService() - .announce( BoxEvent.ON_SESSION_DESTROYED, Struct.of( - Key.session, this - ) ); + // Try to get the application scope from the incoming listener, if not, we can try to go to the Application Service + // If that fails, we can just pass an empty struct + ApplicationService appService = BoxRuntime.getInstance().getApplicationService(); + Application targetApp = listener.getApplication(); + if ( targetApp == null && appService.hasApplication( this.applicationName ) ) { + targetApp = appService.getApplication( this.applicationName ); + } + ApplicationScope targetAppScope = ( targetApp != null ? targetApp.getApplicationScope() : new ApplicationScope() ); - // Any buffer output in this context will be discarded - // Create a temp request context with an application context with our application listener. - // This will allow the application scope to be available as well as all settings from the original Application.bx - listener.onSessionEnd( - new ScriptingRequestBoxContext( - new ApplicationBoxContext( - BoxRuntime.getInstance().getRuntimeContext(), - listener.getApplication() - ), - listener - ), - new Object[] { - // If the session scope is null, just pass an empty struct - sessionScope != null ? sessionScope : Struct.of(), - // Pass the application scope - listener.getApplication().getApplicationScope() - } - ); + try { + // Announce it's destruction to the runtime first + BoxRuntime.getInstance() + .getInterceptorService() + .announce( + BoxEvent.ON_SESSION_DESTROYED, + Struct.of( + Key.session, this, + Key.application, targetAppScope + ) + ); - // Clear the session scope - if ( this.sessionScope != null ) { - this.sessionScope.clear(); + // Any buffer output in this context will be discarded + // Create a temp request context with an application context with our application listener. + // This will allow the application scope to be available as well as all settings from the original Application.bx + listener.onSessionEnd( + listener.getRequestContext(), + new Object[] { + // If the session scope is null, just pass an empty struct + sessionScope != null ? sessionScope : Struct.of(), + // Pass the application scope + targetAppScope + } + ); + } finally { + // Clear the session scope + if ( this.sessionScope != null ) { + this.sessionScope.clear(); + } + this.sessionScope = null; } - this.sessionScope = null; + } /** From 30c666a6b371e6b6ffc0d6c3328f4b9d0723fbee Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 20 Dec 2024 20:33:34 -0600 Subject: [PATCH 044/161] BL-823 --- src/main/antlr/SQLGrammar.g4 | 13 +- src/main/antlr/SQLLexer.g4 | 3 +- .../select/expression/SQLCountFunction.java | 1 - .../operation/SQLBinaryOperation.java | 106 ++++++++- .../operation/SQLBinaryOperator.java | 9 + .../operation/SQLUnaryOperation.java | 59 ++++- .../operation/SQLUnaryOperator.java | 3 + .../compiler/toolchain/SQLVisitor.java | 14 ++ .../runtime/components/system/Lock.java | 2 +- .../dynamic/casters/IntegerCaster.java | 4 +- .../boxlang/runtime/jdbc/QueryParameter.java | 3 +- .../runtime/jdbc/qoq/QoQExecutionService.java | 6 +- .../runtime/jdbc/qoq/QoQFunctionService.java | 4 + .../jdbc/qoq/QoQIntersectionGenerator.java | 4 +- .../jdbc/qoq/functions/scalar/Left.java | 58 +++++ .../jdbc/qoq/functions/scalar/Mod.java | 7 + .../jdbc/qoq/functions/scalar/Power.java | 7 + .../jdbc/qoq/functions/scalar/Right.java | 58 +++++ .../ortus/boxlang/runtime/types/Query.java | 14 ++ .../boxlang/runtime/types/QueryColumn.java | 37 ++- .../runtime/types/QueryColumnType.java | 10 +- src/test/java/external/specs/QoQAliasTest.cfc | 112 +++++++++ src/test/java/external/specs/QoQBasicTest.cfc | 54 +++++ src/test/java/external/specs/QoQCastTest.cfc | 91 ++++++++ .../java/external/specs/QoQColumnNameTest.cfc | 93 ++++++++ .../external/specs/QoQCompatDataTypeTest.cfc | 212 ++++++++++++++++++ .../external/specs/QoQDivideByZeroTest.cfc | 29 +++ .../external/specs/QoQMathEmptyStringTest.cfc | 124 ++++++++++ .../java/external/specs/QoQMathNullTest.cfc | 181 +++++++++++++++ .../external/specs/QoQNullGroupingTest.cfc | 78 +++++++ .../external/specs/QoQSpecialCharsTest.cfc | 125 +++++++++++ .../java/external/specs/QoQUnionParamTest.cfc | 45 ++++ .../specs/QoQValidationChecksTest.cfc | 136 +++++++++++ .../ortus/boxlang/compiler/QoQParseTest.java | 27 +-- 34 files changed, 1682 insertions(+), 47 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Left.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Right.java create mode 100644 src/test/java/external/specs/QoQAliasTest.cfc create mode 100644 src/test/java/external/specs/QoQCastTest.cfc create mode 100644 src/test/java/external/specs/QoQColumnNameTest.cfc create mode 100644 src/test/java/external/specs/QoQCompatDataTypeTest.cfc create mode 100644 src/test/java/external/specs/QoQDivideByZeroTest.cfc create mode 100644 src/test/java/external/specs/QoQMathEmptyStringTest.cfc create mode 100644 src/test/java/external/specs/QoQMathNullTest.cfc create mode 100644 src/test/java/external/specs/QoQNullGroupingTest.cfc create mode 100644 src/test/java/external/specs/QoQSpecialCharsTest.cfc create mode 100644 src/test/java/external/specs/QoQUnionParamTest.cfc create mode 100644 src/test/java/external/specs/QoQValidationChecksTest.cfc diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 8226212f5..620664969 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -227,8 +227,13 @@ expr: | BIND_PARAMETER | (table_name DOT)? column_name | unary_operator expr + // concat operator | expr PIPE2 expr - | expr ( STAR | DIV | MOD) expr + // bitwise operators + | expr (CARET | AMP | PIPE) expr + // math operators + | expr (STAR | DIV | MOD) expr + // math or maybe concat operators | expr (PLUS | MINUS) expr // Special handling of cast to allow cast( foo as number) | CAST_ OPEN_PAR expr AS_ (name | STRING_LITERAL) CLOSE_PAR @@ -571,7 +576,7 @@ recursive_select: unary_operator: MINUS | PLUS - // | TILDE + | TILDE | BANG ; @@ -763,7 +768,7 @@ schema_name: ; table_name: - any_name + IDENTIFIER ; table_or_index_name: @@ -771,7 +776,7 @@ table_or_index_name: ; column_name: - any_name + IDENTIFIER ; collation_name: diff --git a/src/main/antlr/SQLLexer.g4 b/src/main/antlr/SQLLexer.g4 index eb754d2b2..291b77462 100644 --- a/src/main/antlr/SQLLexer.g4 +++ b/src/main/antlr/SQLLexer.g4 @@ -18,6 +18,7 @@ COMMA: ','; ASSIGN: '='; STAR: '*'; PLUS: '+'; +CARET: '^'; MINUS: '-'; TILDE: '~'; PIPE2: '||'; @@ -205,7 +206,7 @@ IDENTIFIER: '"' (~'"' | '""')* '"' | '`' (~'`' | '``')* '`' | '[' ~']'* ']' - | [A-Z_\u007F-\uFFFF] [A-Z_0-9\u007F-\uFFFF]* + | [A-Z$_\u007F-\uFFFF] [A-Z$_0-9\u007F-\uFFFF]* ; NUMERIC_LITERAL: ((DIGIT+ ('.' DIGIT*)?) | ('.' DIGIT+)) ('E' [-+]? DIGIT+)? | '0x' HEX_DIGIT+; diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java index 0fbf42466..6e4e56a8e 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java @@ -105,7 +105,6 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { * Evaluate the expression aginst a partition of data */ public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { - // TODO: handle distinct // If this is count(*), then we just return the number of intersections with no regard to whether there are nulls if ( getArguments().get( 0 ) instanceof SQLStarExpression ) { diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java index 879654822..e92461af0 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java @@ -21,6 +21,8 @@ import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNullLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLStringLiteral; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.dynamic.casters.StringCaster; @@ -232,6 +234,30 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return null; } return leftNum - rightNum; + case BITWISE_AND : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumber( left, QoQExec, intersection ); + rightNum = evalAsNumber( right, QoQExec, intersection ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() & rightNum.intValue(); + case BITWISE_OR : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumber( left, QoQExec, intersection ); + rightNum = evalAsNumber( right, QoQExec, intersection ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() | rightNum.intValue(); + case BITWISE_XOR : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumber( left, QoQExec, intersection ); + rightNum = evalAsNumber( right, QoQExec, intersection ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() ^ rightNum.intValue(); case MODULO : ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); @@ -239,6 +265,9 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { if ( leftNum == null || rightNum == null ) { return null; } + if ( rightNum.doubleValue() == 0 ) { + throw new BoxRuntimeException( "Division by zero" ); + } return leftNum % rightNum; case MULTIPLY : ensureNumericOperands( QoQExec ); @@ -271,7 +300,9 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return false; case PLUS : - if ( left.isNumeric( QoQExec ) && right.isNumeric( QoQExec ) ) { + // If both sides are numeric, or null + // This ensure null + 5 is still math. + if ( ( left.isNumeric( QoQExec ) || left instanceof SQLNullLiteral ) && ( right.isNumeric( QoQExec ) || right instanceof SQLNullLiteral ) ) { leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -343,6 +374,30 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse return null; } return leftNum - rightNum; + case BITWISE_AND : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() & rightNum.intValue(); + case BITWISE_OR : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() | rightNum.intValue(); + case BITWISE_XOR : + ensureNumericOperands( QoQExec ); + leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); + rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); + if ( leftNum == null || rightNum == null ) { + return null; + } + return leftNum.intValue() ^ rightNum.intValue(); case MODULO : ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); @@ -350,6 +405,9 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse if ( leftNum == null || rightNum == null ) { return null; } + if ( rightNum.doubleValue() == 0 ) { + throw new BoxRuntimeException( "Division by zero" ); + } return leftNum % rightNum; case MULTIPLY : ensureNumericOperands( QoQExec ); @@ -449,13 +507,15 @@ private void ensureBooleanOperands( QoQSelectExecution QoQExec ) { /** * Reusable helper method to ensure that the left and right operands are numeric expressions or numeric columns + * nulls are ok in math operations + * Strings we'll let slide too since "" casts to a 0 */ private void ensureNumericOperands( QoQSelectExecution QoQExec ) { - if ( !left.isNumeric( QoQExec ) ) { + if ( !left.isNumeric( QoQExec ) && ! ( left instanceof SQLNullLiteral ) && ! ( left instanceof SQLStringLiteral ) ) { throw new BoxRuntimeException( "Left side of a math [" + operator.getSymbol() + "] operation must be a numeric expression or numeric column. It is [" + left.getClass().getName() + "]" ); } - if ( !right.isNumeric( QoQExec ) ) { + if ( !right.isNumeric( QoQExec ) && ! ( right instanceof SQLNullLiteral ) && ! ( right instanceof SQLStringLiteral ) ) { throw new BoxRuntimeException( "Right side of a math [" + operator.getSymbol() + "] operation must be a numeric expression or numeric column. It is [" + right.getClass().getName() + "]" ); } @@ -470,8 +530,24 @@ private void ensureNumericOperands( QoQSelectExecution QoQExec ) { * * @return */ - private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { - return ( ( Number ) expression.evaluate( QoQExec, intersection ) ).doubleValue(); + private Double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { + Object value = expression.evaluate( QoQExec, intersection ); + Number nValue = 0; + if ( value == null ) { + nValue = null; + } else if ( value instanceof Number n ) { + nValue = n; + } else if ( value instanceof String s ) { + if ( s.isEmpty() ) { + nValue = 0; + } else { + throw new BoxRuntimeException( "Cannot string as a number: [" + s + "]" ); + } + } + if ( nValue == null ) { + return null; + } + return nValue.doubleValue(); } /** @@ -483,8 +559,24 @@ private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExe * * @return */ - private double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecution QoQExec, List intersections ) { - return ( ( Number ) expression.evaluateAggregate( QoQExec, intersections ) ).doubleValue(); + private Double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecution QoQExec, List intersections ) { + Object value = expression.evaluateAggregate( QoQExec, intersections ); + Number nValue = 0; + if ( value == null ) { + nValue = null; + } else if ( value instanceof Number n ) { + nValue = n; + } else if ( value instanceof String s ) { + if ( s.isEmpty() ) { + nValue = 0; + } else { + throw new BoxRuntimeException( "Cannot string as a number: [" + s + "]" ); + } + } + if ( nValue == null ) { + return null; + } + return nValue.doubleValue(); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperator.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperator.java index 75aa6637e..3a008fa3f 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperator.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperator.java @@ -17,6 +17,9 @@ public enum SQLBinaryOperator { PLUS, + BITWISE_AND, + BITWISE_OR, + BITWISE_XOR, MINUS, MULTIPLY, DIVIDE, @@ -67,6 +70,12 @@ public String getSymbol() { return "NOT LIKE"; case CONCAT : return "||"; + case BITWISE_XOR : + return "^"; + case BITWISE_AND : + return "&"; + case BITWISE_OR : + return "|"; default : return ""; } diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java index 319b77392..fa8d54f58 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java @@ -21,6 +21,7 @@ import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNullLiteral; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; @@ -139,6 +140,13 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { case PLUS : ensureNumericOperand( QoQExec ); return expression.evaluate( QoQExec, intersection ); + case BITWISE_NOT : + ensureNumericOperand( QoQExec ); + Number value = evalAsNumber( expression, QoQExec, intersection ); + if ( value == null ) { + return null; + } + return ~value.intValue(); default : throw new BoxRuntimeException( "Unknown binary operator: " + operator ); @@ -149,10 +157,28 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { * Evaluate the expression aginst a partition of data */ public Object evaluateAggregate( QoQSelectExecution QoQExec, List intersections ) { - if ( intersections.isEmpty() ) { - return null; + // Implement each unary operator + switch ( operator ) { + case ISNOTNULL : + return expression.evaluateAggregate( QoQExec, intersections ) != null; + case ISNULL : + return expression.evaluateAggregate( QoQExec, intersections ) != null; + case MINUS : + ensureNumericOperand( QoQExec ); + return -evalAsNumberAggregate( expression, QoQExec, intersections ); + case NOT : + ensureBooleanOperand( QoQExec ); + return ! ( ( boolean ) expression.evaluateAggregate( QoQExec, intersections ) ); + case PLUS : + ensureNumericOperand( QoQExec ); + return expression.evaluateAggregate( QoQExec, intersections ); + case BITWISE_NOT : + ensureNumericOperand( QoQExec ); + return ~evalAsNumberAggregate( expression, QoQExec, intersections ).intValue(); + default : + throw new BoxRuntimeException( "Unknown binary operator: " + operator ); + } - return evaluate( QoQExec, intersections.get( 0 ) ); } /** @@ -172,7 +198,7 @@ private void ensureBooleanOperand( QoQSelectExecution QoQExec ) { * Reusable helper method to ensure that the left and right operands are numeric expressions or numeric columns */ private void ensureNumericOperand( QoQSelectExecution QoQExec ) { - if ( !expression.isNumeric( QoQExec ) ) { + if ( !expression.isNumeric( QoQExec ) && ! ( expression instanceof SQLNullLiteral ) ) { throw new BoxRuntimeException( "Unary operation [" + operator.getSymbol() + "] must be a numeric expression or numeric column" ); } } @@ -186,8 +212,29 @@ private void ensureNumericOperand( QoQSelectExecution QoQExec ) { * * @return */ - private double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { - return ( ( Number ) expression.evaluate( QoQExec, intersection ) ).doubleValue(); + private Double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExec, int[] intersection ) { + Number value = ( Number ) expression.evaluate( QoQExec, intersection ); + if ( value == null ) { + return null; + } + return value.doubleValue(); + } + + /** + * Helper for evaluating an expression as a number + * + * @param tableLookup + * @param expression + * @param i + * + * @return + */ + private Double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecution QoQExec, List intersections ) { + Number value = ( Number ) expression.evaluateAggregate( QoQExec, intersections ); + if ( value == null ) { + return null; + } + return value.doubleValue(); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperator.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperator.java index 221811291..53cdba912 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperator.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperator.java @@ -19,6 +19,7 @@ public enum SQLUnaryOperator { PLUS, MINUS, NOT, + BITWISE_NOT, ISNULL, ISNOTNULL; @@ -34,6 +35,8 @@ public String getSymbol() { return "IS NULL"; case ISNOTNULL : return "IS NOT NULL"; + case BITWISE_NOT : + return "~"; default : return ""; } diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index 719ffadc9..354ad22bd 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -498,6 +498,12 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j arguments = ctx.expr().stream().map( e -> visitExpr( e, table, joins ) ).toList(); } if ( functionName.equals( Key.count ) ) { + if ( arguments.size() == 0 ) { + tools.reportError( "COUNT() must have at least one argument", pos ); + } + if ( arguments.size() > 1 ) { + tools.reportError( "COUNT() can only have one argument", pos ); + } return new SQLCountFunction( functionName, arguments, hasDistinct, pos, src ); } else { return new SQLFunction( functionName, arguments, pos, src ); @@ -521,6 +527,12 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.PLUS, pos, src, table, joins ); } else if ( ctx.MINUS() != null ) { return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.MINUS, pos, src, table, joins ); + } else if ( ctx.PIPE() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.BITWISE_OR, pos, src, table, joins ); + } else if ( ctx.AMP() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.BITWISE_AND, pos, src, table, joins ); + } else if ( ctx.CARET() != null ) { + return binarySimple( ctx.expr( 0 ), ctx.expr( 1 ), SQLBinaryOperator.BITWISE_XOR, pos, src, table, joins ); } else if ( ctx.OPEN_PAR() != null ) { // Needs to run AFTER function and IN checks return new SQLParenthesis( visitExpr( ctx.expr( 0 ), table, joins ), pos, src ); @@ -534,6 +546,8 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j op = SQLUnaryOperator.MINUS; } else if ( ctx.unary_operator().PLUS() != null ) { op = SQLUnaryOperator.PLUS; + } else if ( ctx.unary_operator().TILDE() != null ) { + op = SQLUnaryOperator.BITWISE_NOT; } else { throw new UnsupportedOperationException( "Unimplemented unary operator: " + ctx.unary_operator().getText() ); } diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Lock.java b/src/main/java/ortus/boxlang/runtime/components/system/Lock.java index bffe0b612..1cc292b12 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Lock.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Lock.java @@ -136,7 +136,7 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod throw new LockException( "Timeout of [" + timeout + "] seconds reached while waiting to acquire lock [" + lockName + "]", lockName, "timeout" ); } else { - // No need to release anything, because we never aquired it!s + // No need to release anything, because we never aquired it! return DEFAULT_RETURN; } } diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/IntegerCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/IntegerCaster.java index c33f2d340..a1d4584ba 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/IntegerCaster.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/IntegerCaster.java @@ -78,7 +78,7 @@ public static Integer cast( Object object, Boolean fail ) { String theValue = StringCaster.cast( object, fail ); if ( theValue == null ) { if ( fail ) { - throw new BoxCastException( String.format( "Can't cast %s to a int.", theValue ) ); + throw new BoxCastException( String.format( "Can't cast [%s] to a int.", theValue ) ); } else { return null; } @@ -87,7 +87,7 @@ public static Integer cast( Object object, Boolean fail ) { return Integer.valueOf( theValue ); } if ( fail ) { - throw new BoxCastException( String.format( "Can't cast %s to a int.", theValue ) ); + throw new BoxCastException( String.format( "Can't cast [%s] to a int.", theValue ) ); } else { return null; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index 9ea1d07c8..84b5cf5ee 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -79,7 +79,7 @@ public class QueryParameter { *
  • `scale` - The scale of the parameter, used only on `double` and `decimal` types. Defaults to `null`.
    • */ private QueryParameter( IStruct param ) { - String sqltype = ( String ) param.getOrDefault( Key.sqltype, "VARCHAR" ); + String sqltype = ( String ) param.getOrDefault( Key.sqltype, param.getOrDefault( Key.type, "VARCHAR" ) ); // allow nulls and null this.isNullParam = BooleanCaster.cast( param.getOrDefault( Key.nulls, param.getOrDefault( Key.nulls2, false ) ) ); this.isListParam = BooleanCaster.cast( param.getOrDefault( Key.list, false ) ); @@ -134,6 +134,7 @@ public Object toSQLType( IBoxContext context ) { case QueryColumnType.CHAR, VARCHAR -> StringCaster.cast( this.value ); case QueryColumnType.BINARY -> this.value; // @TODO: Will this work? case QueryColumnType.BIT -> BooleanCaster.cast( this.value ); + case QueryColumnType.BOOLEAN -> BooleanCaster.cast( this.value ); case QueryColumnType.TIME -> DateTimeCaster.cast( this.value, context ); case QueryColumnType.DATE -> DateTimeCaster.cast( this.value, context ); case QueryColumnType.TIMESTAMP -> new java.sql.Timestamp( DateTimeCaster.cast( this.value, context ).toEpochMillis() ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index 8d93f8a29..cb8117e0b 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -69,9 +69,6 @@ public class QoQExecutionService { * @return the AST */ public static SQLNode parseSQL( String sql ) { - if ( sql.indexOf( "bradtest" ) > -1 ) { - // System.out.println( sql ); - } DynamicObject trans = frTransService.startTransaction( "BL QoQ Parse", "" ); SQLParser parser = new SQLParser(); ParsingResult result; @@ -302,7 +299,8 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t StringBuilder sb = new StringBuilder(); for ( SQLExpression expression : groupBys ) { // TODO: hash large values - sb.append( StringCaster.cast( expression.evaluate( QoQExec, intersection ) ) ); + Object cellValue = expression.evaluate( QoQExec, intersection ); + sb.append( StringCaster.cast( cellValue == null ? "<>" : cellValue ) ); } partitionKey = sb.toString(); } else { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index 7efaa5c0a..d7fba8ac1 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -39,11 +39,13 @@ import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Exp; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Floor; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.IsNull; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Left; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Length; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Lower; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Ltrim; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Mod; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Power; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Right; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Rtrim; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Sin; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Sqrt; @@ -88,6 +90,8 @@ public class QoQFunctionService { register( Tan.INSTANCE ); register( Trim.INSTANCE ); register( Upper.INSTANCE ); + register( Left.INSTANCE ); + register( Right.INSTANCE ); // Aggregate register( Max.INSTANCE ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java index 0cf925e38..2b128dbab 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java @@ -184,7 +184,7 @@ private static Stream handleFullOuterJoin( Stream theStream, Query newIntersection[ i.length ] = j; return newIntersection; } ) - .filter( j -> ( Boolean ) joinOn.evaluate( QoQExec, j ) ); + .filter( j -> joinOn == null || ( Boolean ) joinOn.evaluate( QoQExec, j ) ); List newStreamList = newStream.collect( Collectors.toList() ); if ( newStreamList.isEmpty() ) { int[] leftOnlyIntersection = Arrays.copyOf( i, i.length + 1 ); @@ -205,7 +205,7 @@ private static Stream handleFullOuterJoin( Stream theStream, Query newIntersection[ i.length ] = j[ 0 ]; return newIntersection; } ) - .filter( joint -> ( Boolean ) joinOn.evaluate( QoQExec, joint ) ); + .filter( joint -> joinOn == null || ( Boolean ) joinOn.evaluate( QoQExec, joint ) ); List newStreamList = newStream.collect( Collectors.toList() ); if ( newStreamList.isEmpty() ) { int[] rightOnlyIntersection = new int[ leftRows.get( 0 ).length + 1 ]; diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Left.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Left.java new file mode 100644 index 000000000..364c5ca03 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Left.java @@ -0,0 +1,58 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Left extends QoQScalarFunctionDef { + + private static final Key name = Key.of( "left" ); + + public static final QoQScalarFunctionDef INSTANCE = new Left(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + return QueryColumnType.VARCHAR; + } + + @Override + public int getMinArgs() { + return 2; + } + + @Override + public Object apply( List args, List expressions ) { + String value = StringCaster.cast( args.get( 0 ) ); + int length = IntegerCaster.cast( args.get( 1 ) ); + if ( length > value.length() ) { + length = value.length(); + } + return value.substring( 0, length ); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java index 15cc2c3a7..f2a8705b7 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Mod.java @@ -51,6 +51,13 @@ public Object apply( List args, List expressions ) { if ( left == null || right == null ) { return null; } + // empty strings are treated as 0 in sql math + if ( left instanceof String s && s.isEmpty() ) { + left = 0; + } + if ( right instanceof String s && s.isEmpty() ) { + right = 0; + } // Modulus does its own casting return Modulus.invoke( left, right ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java index 5662ccb2f..ce74f8595 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Power.java @@ -51,6 +51,13 @@ public Object apply( List args, List expressions ) { if ( left == null || right == null ) { return null; } + // empty strings are treated as 0 in sql math + if ( left instanceof String s && s.isEmpty() ) { + left = 0; + } + if ( right instanceof String s && s.isEmpty() ) { + right = 0; + } return Math.pow( NumberCaster.cast( left ).doubleValue(), NumberCaster.cast( right ).doubleValue() ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Right.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Right.java new file mode 100644 index 000000000..c2e2fbab5 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Right.java @@ -0,0 +1,58 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import java.util.List; + +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.QueryColumnType; + +public class Right extends QoQScalarFunctionDef { + + private static final Key name = Key.of( "right" ); + + public static final QoQScalarFunctionDef INSTANCE = new Right(); + + @Override + public Key getName() { + return name; + } + + @Override + public QueryColumnType getReturnType( QoQSelectExecution QoQExec, List expressions ) { + return QueryColumnType.VARCHAR; + } + + @Override + public int getMinArgs() { + return 2; + } + + @Override + public Object apply( List args, List expressions ) { + String value = StringCaster.cast( args.get( 0 ) ); + int length = IntegerCaster.cast( args.get( 1 ) ); + if ( length > value.length() ) { + length = value.length(); + } + return value.substring( value.length() - length ); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index 3a218de7c..ec0ea82e1 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -343,6 +343,20 @@ public QueryColumn getColumn( Key name ) { return column; } + /** + * Get the QueryColumn object for a column + * Throws an exception if the column doesn't exist + * + * This method for CF/Lucee compat + * + * @param name column name + * + * @return QueryColumn object + */ + public QueryColumn getColumn( String name ) { + return getColumn( Key.of( name ) ); + } + /** * Get data for a row as an array. 0-based index! * Array is passed by reference and changes made to it will be reflected in the diff --git a/src/main/java/ortus/boxlang/runtime/types/QueryColumn.java b/src/main/java/ortus/boxlang/runtime/types/QueryColumn.java index 337f77f74..cd5afa8ac 100644 --- a/src/main/java/ortus/boxlang/runtime/types/QueryColumn.java +++ b/src/main/java/ortus/boxlang/runtime/types/QueryColumn.java @@ -146,6 +146,19 @@ public Object getCell( int row ) { return this.query.getData().get( row )[ index ]; } + /** + * Get the value of a cell in this column + * + * This method for CF/Lucee compat + * + * @param row The row to get, 0-based index + * + * @return The value of the cell + */ + public Object get( int row, Object defaultValue ) { + return getCell( row ); + } + /** * Get all data in a column as a Java Object[] * Data is copied, so re-assignments into the array will not be reflected in the query. @@ -222,7 +235,13 @@ public Object dereference( IBoxContext context, Key name, Boolean safe ) { } // If dereferencing a query column with a NON number like qry.col["key"], then we get the value at the "current" row and dererence it - return Referencer.get( context, getCell( query.getRowFromContext( context ) ), name, safe ); + int row = query.getRowFromContext( context ); + Object cellValue = getCell( row ); + if ( cellValue == null ) { + throw new BoxRuntimeException( + "Cannot dereference the key [" + name.getName() + "()] on the null value in row " + ( row + 1 ) + " of column [" + this.name.getName() + "]" ); + } + return Referencer.get( context, cellValue, name, safe ); } @@ -230,14 +249,26 @@ public Object dereference( IBoxContext context, Key name, Boolean safe ) { public Object dereferenceAndInvoke( IBoxContext context, Key name, Object[] positionalArguments, Boolean safe ) { // qry.col.method() will ALWAYS get the value from the current row and call the method on that cell value // Unlike Lucee/Adobe, we'll never call the method on the query column itself - return DynamicInteropService.invoke( context, getCell( query.getRowFromContext( context ) ), name.getName(), safe, positionalArguments ); + int row = query.getRowFromContext( context ); + Object cellValue = getCell( row ); + if ( cellValue == null ) { + throw new BoxRuntimeException( + "Cannot invoke method [" + name.getName() + "()] on the null value in row " + ( row + 1 ) + " of column [" + this.name.getName() + "]" ); + } + return DynamicInteropService.invoke( context, cellValue, name.getName(), safe, positionalArguments ); } @Override public Object dereferenceAndInvoke( IBoxContext context, Key name, Map namedArguments, Boolean safe ) { // qry.col.method() will ALWAYS get the value from the current row and call the method on that cell value // Unlike Lucee/Adobe, we'll never call the method on the query column itself - return DynamicInteropService.invoke( context, getCell( query.getRowFromContext( context ) ), name.getName(), safe, namedArguments ); + int row = query.getRowFromContext( context ); + Object cellValue = getCell( row ); + if ( cellValue == null ) { + throw new BoxRuntimeException( + "Cannot invoke method [" + name.getName() + "()] on the null value in row " + ( row + 1 ) + " of column [" + this.name.getName() + "]" ); + } + return DynamicInteropService.invoke( context, cellValue, name.getName(), safe, namedArguments ); } @Override diff --git a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java index c44af6b37..77d3dc708 100644 --- a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java +++ b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java @@ -26,6 +26,7 @@ public enum QueryColumnType { BIGINT( Types.BIGINT ), BINARY( Types.BINARY ), + BOOLEAN( Types.BOOLEAN ), BIT( Types.BIT ), CHAR( Types.CHAR ), DATE( Types.DATE ), @@ -68,8 +69,10 @@ public static QueryColumnType fromString( String type ) { case "nclob" : return OBJECT; case "bit" : - case "boolean" : return BIT; + case "boolean" : + case "bool" : + return BOOLEAN; case "nchar" : case "char" : return CHAR; @@ -85,6 +88,7 @@ public static QueryColumnType fromString( String type ) { case "float" : case "double" : case "numeric" : + case "number" : return DOUBLE; case "idstamp" : return CHAR; @@ -148,6 +152,8 @@ public String toString() { return "other"; case NULL : return "null"; + case BOOLEAN : + return "boolean"; default : throw new IllegalArgumentException( "Unknown QueryColumnType: " + this ); } @@ -173,7 +179,7 @@ public static QueryColumnType fromSQLType( int type ) { case Types.BLOB : return OBJECT; case Types.BOOLEAN : - return BIT; + return BOOLEAN; case Types.CHAR : return VARCHAR; case Types.CLOB : diff --git a/src/test/java/external/specs/QoQAliasTest.cfc b/src/test/java/external/specs/QoQAliasTest.cfc new file mode 100644 index 000000000..f9e649d41 --- /dev/null +++ b/src/test/java/external/specs/QoQAliasTest.cfc @@ -0,0 +1,112 @@ +/** +* Copied from test\tickets\LDEV4592.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, testBox ){ + + describe( "test hsqldb qoq support ", function(){ + + it( "test qoq different column types, duplicate columns ", function(){ + + var news = queryNew("id,title","bit,varchar",[ // note bit + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + var news2 = queryNew("id,title", "integer,varchar",[ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + // throws [incompatible data type in operation], but if you alias the second title column, we load a subset of data and it works + query name="local.q" dbtype="query" { + echo("SELECT news.title, news2.title FROM news, news2"); // duplicate column names + } + + expect ( q.recordcount ).toBe( 4 ); + }); + + it( "test qoq different column types, aliased duplicate columns ", function(){ + + var news = queryNew("id,title","bit,varchar",[ // note bit + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + var news2 = queryNew("id,title", "integer,varchar",[ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + // throws [incompatible data type in operation] + query name="local.q" dbtype="query" { + echo("SELECT news.title, news2.title as t2 FROM news, news2"); // aliased column names + } + + expect ( q.recordcount ).toBe( 4 ); + }); + + it( "test qoq same column types, aliased duplicate column names ", function(){ + + var news = queryNew("id,title","bit,varchar",[ // note bit + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + var news2 = queryNew("id,title", "integer,varchar",[ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + // throws [incompatible data type in operation] + query name="local.q" dbtype="query" { + echo("SELECT news.title, news2.title as t2 FROM news, news2"); // note alias + } + + expect ( q.recordcount ).toBe( 4 ); + }); + + it( "test qoq different column types, duplicate column names ", function(){ + + var news = queryNew("id,title","integer,varchar", [ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + var news2 = queryNew("id,title", "integer,varchar",[ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + query name="local.q" dbtype="query" { + echo("SELECT news.title, news2.title FROM news, news2"); // duplicate column names + } + + expect ( q.recordcount ).toBe( 4 ); + }); + + it( "test qoq different column types, aliased duplicate column names ", function(){ + + var news = queryNew("id,title","integer,varchar", [ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + var news2 = queryNew("id,title", "integer,varchar",[ + {"id":1,"title":"Dewey defeats Truman"}, + {"id":2,"title":"Man walks on Moon"} + ]); + + query name="local.q" dbtype="query" { + echo("SELECT news.title, news2.title as t2 FROM news, news2"); // note alias + } + + expect ( q.recordcount ).toBe( 4 ); + }); + + } ); + } + +} + diff --git a/src/test/java/external/specs/QoQBasicTest.cfc b/src/test/java/external/specs/QoQBasicTest.cfc index accbc6159..508495b9c 100644 --- a/src/test/java/external/specs/QoQBasicTest.cfc +++ b/src/test/java/external/specs/QoQBasicTest.cfc @@ -2,6 +2,8 @@ * Copied from test\tickets\LDEV3042.cfc in Lucee * and test\tickets\LDEV4627.cfc * and test\tickets\LDEV4641.cfc +* and test\tickets\LDEV4695.cfc +* and test\tickets\LDEV4691.cfc */ component extends="testbox.system.BaseSpec"{ @@ -24,6 +26,7 @@ component extends="testbox.system.BaseSpec"{ [ 'Amy Merryweather',58,'Amy@company.com','Executive',false,12,2,createDate(2008,3,21),true,nullValue(),'PURPLE' ] ] ); } + function run( testResults , testBox ) { describe( 'QofQ' , function(){ @@ -633,6 +636,57 @@ component extends="testbox.system.BaseSpec"{ }); + it( title='QoQ error with types and subselect' , body=function() { + var q = queryNew("navid, type, url", "varchar,decimal,VarChar", + [{ + "navid": 200, + "type": 1, + "url": "football" + } + ] + ); + query name="local.res" dbtype="query" { + echo(" + select navid, type, url + from local.q + where url = 'football' + and type = 1 + and left( navid, 3 ) in ( + select navid + from local.q + where type = 1 + and url = 'football') + "); + } + expect( res.recordcount ).toBe( 1 ); + }); + + it( title='QoQ check decimal types scale are preserved' , body=function() { + var dec = 5.57; + var q1 = QueryNew( "id,dec", "integer,decimal" ); + q1.addRow( { id: 1, dec: dec } ); + var q2 = QueryNew( "id,str", "integer,varchar" ); + q2.addRow( { id: 1, str: "testing" } ); + + var q3sql = " + select q1.* + from q1, q2 + where q1.id = q2.id + "; + var q3 = QueryExecute( q3sql, {}, {dbtype: "query"} ); + // q3.dec should be 5.57. It was 6 instead. + expect( q3.dec ).toBe( dec ); + + var q4sql = " + select * + from q1, q2 + where q1.id = q2.id + "; + var q4 = QueryExecute( q4sql, {}, {dbtype: "query"} ); + // q4.dec should be 5.57. It was 6 instead. + expect( q4.dec ).toBe( dec ); + }); + }); } diff --git a/src/test/java/external/specs/QoQCastTest.cfc b/src/test/java/external/specs/QoQCastTest.cfc new file mode 100644 index 000000000..5a628d8ca --- /dev/null +++ b/src/test/java/external/specs/QoQCastTest.cfc @@ -0,0 +1,91 @@ +/** +* Copied from test\tickets\_LDEV3615.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + function run( testResults , testBox ) { + + describe("testcase for LDEV-3522", function(){ + + it(title="Can cast as date", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[['1/1/2025']]); + var actual = queryExecute( + "SELECT cast( foo as date ) as asDate, + convert( foo, date ) as asDate2, + convert( foo, 'date' ) as asDate3 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.asDate ).toBeDate(); + expect( actual.asDate ).toBeInstanceOf( 'DateTime' ); + expect( actual.asDate2 ).toBeDate(); + expect( actual.asDate2 ).toBeInstanceOf( 'DateTime' ); + expect( actual.asDate3 ).toBeDate(); + expect( actual.asDate3 ).toBeInstanceOf( 'DateTime' ); + }); + + it(title="Can cast as string", body=function( currentSpec ){ + qry = QueryNew('foo','date',[[now()]]); + var actual = queryExecute( + "SELECT foo, + cast( foo as string ) as asString, + convert( foo, string ) as asString2, + convert( foo, 'string' ) as asString3 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.foo ).toBeDate(); + expect( actual.foo ).toBeInstanceOf( 'DateTime' ); + expect( actual.asString ).toBeString(); + expect( actual.asString ).toBeInstanceOf( 'java.lang.String' ); + expect( actual.asString2 ).toBeString(); + expect( actual.asString2 ).toBeInstanceOf( 'java.lang.String' ); + expect( actual.asString3 ).toBeString(); + expect( actual.asString3 ).toBeInstanceOf( 'java.lang.String' ); + }); + + it(title="Can cast as number", body=function( currentSpec ){ + qry = QueryNew('foo','string',[['40']]); + var actual = queryExecute( + "SELECT foo, + cast( foo as number ) as asNumber, + convert( foo, number ) as asNumber2, + convert( foo, 'number' ) as asNumber3 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.foo ).toBeString(); + expect( actual.foo ).toBeInstanceOf( 'java.lang.String' ); + expect( actual.asNumber ).toBeNumeric(); + expect( actual.asNumber ).toBeInstanceOf( 'java.lang.Number' ); + expect( actual.asNumber2 ).toBeNumeric(); + expect( actual.asNumber2 ).toBeInstanceOf( 'java.lang.Number' ); + expect( actual.asNumber3 ).toBeNumeric(); + expect( actual.asNumber3 ).toBeInstanceOf( 'java.lang.Number' ); + }); + + it(title="Can cast as boolean", body=function( currentSpec ){ + qry = QueryNew('foo','string',[['true']]); + var actual = queryExecute( + "SELECT foo, + cast( foo as boolean ) as asBoolean, + convert( foo, boolean ) as asBoolean2, + convert( foo, 'bool' ) as asBool3 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.foo ).toBeString(); + expect( actual.foo ).toBeInstanceOf( 'java.lang.String' ); + expect( actual.asBoolean ).toBeBoolean(); + expect( actual.asBoolean ).toBeInstanceOf( 'java.lang.Boolean' ); + expect( actual.asBoolean2 ).toBeBoolean(); + expect( actual.asBoolean2 ).toBeInstanceOf( 'java.lang.Boolean' ); + expect( actual.asBool3 ).toBeBoolean(); + expect( actual.asBool3 ).toBeInstanceOf( 'java.lang.Boolean' ); + }); + + + }); + + } +} + diff --git a/src/test/java/external/specs/QoQColumnNameTest.cfc b/src/test/java/external/specs/QoQColumnNameTest.cfc new file mode 100644 index 000000000..5f4896057 --- /dev/null +++ b/src/test/java/external/specs/QoQColumnNameTest.cfc @@ -0,0 +1,93 @@ +/** +* Copied from test\tickets\_LDEV3615.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + + function testQoQunion (){ + var q1 = queryNew( + "subtype", + "varchar", + [["RECORD3_TEMPLATE"]] + ); + var q2 = queryNew( + "id", + "int", + [[1]] + ); + + ``` + + SELECT subtype, + subtype AS subject + FROM q1 + UNION + SELECT NULL AS subtype, + NULL AS subject + FROM q2 + + + ``` + // remove white space + expect( serializeJson( local.result ).reReplace( '\s', '', 'all' ) ).toBe('{"COLUMNS":["subtype","subject"],"DATA":[["RECORD3_TEMPLATE","RECORD3_TEMPLATE"],[null,null]]}'); + } + + function testNullAliases (){ + var q1 = queryNew( "col" ) + + ``` + + SELECT NULL AS a, + NULL AS b + FROM q1 + + + ``` + expect( local.result.columnList ).toBe('A,B'); + } + + function testNullNoAlias (){ + var q1 = queryNew( "col" ) + + ``` + + SELECT NULL, + NULL + FROM q1 + + + ``` + expect( local.result.columnList ).toBe('COLUMN_0,COLUMN_1'); + } + + function testBooleanAliases (){ + var q1 = queryNew( "col" ) + + ``` + + SELECT true AS a, + true AS b, + false AS c, + false AS d + FROM q1 + + + ``` + expect( local.result.columnList ).toBe('A,B,C,D'); + } + + function testBooleanNoAlias (){ + var q1 = queryNew( "col" ) + + ``` + + SELECT true, + false + FROM q1 + + + ``` + expect( local.result.columnList ).toBe('COLUMN_0,COLUMN_1'); + } +} + diff --git a/src/test/java/external/specs/QoQCompatDataTypeTest.cfc b/src/test/java/external/specs/QoQCompatDataTypeTest.cfc new file mode 100644 index 000000000..484c37751 --- /dev/null +++ b/src/test/java/external/specs/QoQCompatDataTypeTest.cfc @@ -0,0 +1,212 @@ +/** +* Copied from test\tickets\LDEV4613.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults , testBox ) { + + describe( title='QofQ' , body=function(){ + + var arrG4AppLicenses = [ + { + "g4app_id": 4, + "g4app_code": "g4emp", + "is_hr": 0, + "is_emp": 1, + "license_min_summed": 0, + "license_max_summed": 400 + } + ]; + + variables.qryG4AppLicenses = QueryNew( + "g4app_id,g4app_code,is_hr,is_emp,license_min_summed,license_max_summed", + "integer,varchar,bit,bit,double,double", + arrG4AppLicenses + ); + + // Setup query #2 + var arrUsageCount = [ + { + "g4app_id": 3, + "g4app_code": "g4hr", + "g4app_name": "HR Management System", + "usage_count": 9 + } + ]; + + variables.qryUsageCount = QueryNew( + "g4app_id,g4app_code,g4app_name,usage_count", + "integer,varchar,varchar,double", + arrUsageCount + ); + + it( title='LDEV-4613 incompatible data type in operation simple join, all upper case sql' , body=function() { + var q = QueryExecute( + sql=" + SELECT QRYG4APPLICENSES.* + FROM QRYUSAGECOUNT, QRYG4APPLICENSES + WHERE QRYUSAGECOUNT.G4APP_ID = QRYG4APPLICENSES.G4APP_ID + ", + options={ + dbtype='query' + } + ); + expect( q ).toBeQuery(); + expect( q.recordcount ).toBe( 0 ); + }); + + it( title='LDEV-4613 incompatible data type in operation simple join, mixed case sql' , body=function() { + + var q = QueryExecute( + sql=" + select qryG4AppLicenses.* + from qryUsageCount, qryG4AppLicenses + where qryUsageCount.g4app_id = qryG4AppLicenses.g4app_id + ", + options={ + dbtype='query' + } + ); + + expect( q ).toBeQuery(); + expect( q.recordcount ).toBe( 0 ); + }); + + it( title='LDEV-4613 incompatible data type in operation simple join, mixed case sql' , body=function() { + + var q = QueryExecute( + sql=" + select qryG4AppLicenses.g4app_id + from qryUsageCount, qryG4AppLicenses + where qryUsageCount.g4app_id = qryG4AppLicenses.g4app_id + ", + options={ + dbtype='query' + } + ); + + expect( q ).toBeQuery(); + expect( q.recordcount ).toBe( 0 ); + }); + + it( title='LDEV-4613 incompatible data type in operation simple join, mixed case sql' , body=function() { + + var q = QueryExecute( + sql=" + select qryG4AppLicenses.G4APP_ID + from qryUsageCount, qryG4AppLicenses + where qryUsageCount.g4app_id = qryG4AppLicenses.g4app_id + ", + options={ + dbtype='query' + } + ); + + expect( q ).toBeQuery(); + expect( q.recordcount ).toBe( 0 ); + }); + + it( title='LDEV-4613 incompatible data type in operation simple join, orig example' , body=function() { + var arrG4AppLicenses = [ + { + "g4app_id": 4, + "g4app_code": "g4emp", + "is_hr": 0, + "is_emp": 1, + "license_min_summed": 0, + "license_max_summed": 400 + }, + { + "g4app_id": 6, + "g4app_code": "g4research", + "is_hr": 1, + "is_emp": 0, + "license_min_summed": 0, + "license_max_summed": 400 + } + ]; + + var qryG4AppLicenses = QueryNew( + "g4app_id,g4app_code,is_hr,is_emp,license_min_summed,license_max_summed", + "integer,varchar,bit,bit,double,double", + arrG4AppLicenses + ); + + // Setup query #2 + var arrUsageCount = [ + { + "g4app_id": 3, + "g4app_code": "g4hr", + "g4app_name": "HR Management System", + "usage_count": 9 + }, + { + "g4app_id": 4, + "g4app_code": "g4emp", + "g4app_name": "Gen4 Employees Center", + "usage_count": 8 + }, + { + "g4app_id": 14, + "g4app_code": "g4benadminportal", + "g4app_name": "Ben Admin Portal", + "usage_count": 11 + }, + { + "g4app_id": 13, + "g4app_code": "g4communicationportal", + "g4app_name": "Communication Portal", + "usage_count": 4 + }, + { + "g4app_id": 5, + "g4app_code": "g4benefits", + "g4app_name": "Gen4Benefits Central", + "usage_count": 6 + }, + { + "g4app_id": 3, + "g4app_code": "g4benadminportal", + "g4app_name": "Ben Admin Portal", + "usage_count": 11 + }, + { + "g4app_id": 7, + "g4app_code": "g4communicationportal", + "g4app_name": "Communication Portal", + "usage_count": 3 + }, + { + "g4app_id": 7, + "g4app_code": "g4hrlite", + "g4app_name": "HR Communication Center", + "usage_count": 4 + } + ]; + + var qryUsageCount = QueryNew( + "g4app_id,g4app_code,g4app_name,usage_count", + "integer,varchar,varchar,double", + arrUsageCount + ); + var q = QueryExecute( + sql=" + select qryG4AppLicenses.* + from qryUsageCount, qryG4AppLicenses + where qryUsageCount.g4app_id = qryG4AppLicenses.g4app_id + ", + options={ + dbtype='query' + } + ); + expect( q ).toBeQuery(); + expect( q.recordcount ).toBe( 1 ); + }); + + + }); + + } + +} + diff --git a/src/test/java/external/specs/QoQDivideByZeroTest.cfc b/src/test/java/external/specs/QoQDivideByZeroTest.cfc new file mode 100644 index 000000000..2be4894ca --- /dev/null +++ b/src/test/java/external/specs/QoQDivideByZeroTest.cfc @@ -0,0 +1,29 @@ +/** +* Copied from test\tickets\LDEV3735.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + describe("testcase for LDEV-3735", function(){ + + it(title="Checking QoQ throws on divide by zero", body=function( currentSpec ){ + var qry = QueryNew('foo','integer',[[40]]); + expect( ()=>queryExecute("SELECT 5/0 As inf From qry", {}, {dbType:"query"}) ).toThrow(); + }); + + it(title="Checking QoQ throws on divide by zero with modulus", body=function( currentSpec ){ + var qry = QueryNew('foo','integer',[[40]]); + expect( ()=>queryExecute("SELECT 5%0 As inf From qry", {}, {dbType:"query"}) ).toThrow(); + }); + + it(title="Checking QoQ throws on divide by zero has useful message", body=function( currentSpec ){ + var qry = QueryNew('foo,bar','integer,integer',[[40,50],[50,100],[10,0]]); + expect( ()=>queryExecute("SELECT foo/isNull(bar,2) As inf From qry", {}, {dbType:"query"}) ).toThrow(); + expect( ()=>queryExecute("SELECT max(foo)/min(bar) As inf From qry", {}, {dbType:"query"}) ).toThrow(); + }); + + }); + } + +} + diff --git a/src/test/java/external/specs/QoQMathEmptyStringTest.cfc b/src/test/java/external/specs/QoQMathEmptyStringTest.cfc new file mode 100644 index 000000000..f0f63c4af --- /dev/null +++ b/src/test/java/external/specs/QoQMathEmptyStringTest.cfc @@ -0,0 +1,124 @@ +/** +* Copied from test\tickets\LDEV3736.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + + describe("testcase for LDEV-3736", function(){ + + it(title="Arithmetic addition with empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 5+5 AS result, + ''+5 as result2, + 5+'' as result3, + ''+'' as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBe( 5 ); + expect( actual.result3 ).toBe( 5 ); + expect( actual.result4 ).toBe( '' ); + }); + + it(title="Arithmetic subtraction empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 20-10 AS result, + ''-5 as result2, + 5-'' as result3, + ''-'' as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBe( -5 ); + expect( actual.result3 ).toBe( 5 ); + expect( actual.result4 ).toBe( 0 ); + }); + + it(title="Arithmetic multiplication empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 2*5 AS result, + ''*5 as result2, + 5*'' as result3, + ''*'' as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBe( 0 ); + expect( actual.result3 ).toBe( 0 ); + expect( actual.result4 ).toBe( 0 ); + }); + + it(title="Arithmetic division empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 20/2 AS result, + ''/5 as result2 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBe( 0 ); + }); + + it(title="Arithmetic bitwise empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 4^2 AS result, + ''^5 as result2, + 5^'' as result3, + ''^'' as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 6 ); + expect( actual.result2 ).toBe( 5 ); + expect( actual.result3 ).toBe( 5 ); + expect( actual.result4 ).toBe( 0 ); + }); + + it(title="Arithmetic modulus empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + // Note % and mod() have different implemntations + var actual = queryExecute( + "SELECT 21%11 AS result, + ''%5 as result2, + mod( 21, 11 ) AS result3, + mod( '', 5 ) as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBe( 0 ); + expect( actual.result3 ).toBe( 10 ); + expect( actual.result4 ).toBe( 0 ); + }); + + it(title="Arithmetic exponent empty string in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT power( 4, 2 ) AS result, + power( '', 5 ) as result2, + power( 5, '' ) as result3, + power( '', '' ) as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 16 ); + expect( actual.result2 ).toBe( 0 ); + expect( actual.result3 ).toBe( 1 ); + expect( actual.result4 ).toBe( 1 ); + }); + + }); + +} + +} + diff --git a/src/test/java/external/specs/QoQMathNullTest.cfc b/src/test/java/external/specs/QoQMathNullTest.cfc new file mode 100644 index 000000000..7258feab2 --- /dev/null +++ b/src/test/java/external/specs/QoQMathNullTest.cfc @@ -0,0 +1,181 @@ +/** +* Copied from test\tickets\LDEV3734.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, textbox ) { + + describe("testcase for LDEV-3734", function(){ + + it(title="Arithmetic addition with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 5+5 AS result, + NULL+5 as result2, + 5+NULL as result3, + NULL+NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic subtraction with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 20-10 AS result, + NULL-5 as result2, + 5-NULL as result3, + NULL-NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic multiplication with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 2*5 AS result, + NULL*5 as result2, + 5*NULL as result3, + NULL*NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic division with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 20/2 AS result, + NULL/5 as result2, + 5/NULL as result3, + NULL/NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic bitwise AND with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 5&3 AS result, + NULL&5 as result2, + 5&NULL as result3, + NULL&NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 1 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic bitwise OR with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 5|3 AS result, + NULL|5 as result2, + 5|NULL as result3, + NULL|NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 7 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic bitwise NOT with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT ~5 AS result, + ~NULL as result2 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( -6 ); + expect( actual.result2 ).toBeNull(); + }); + + it(title="Arithmetic bitwise XOR with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT 4^2 AS result, + NULL^5 as result2, + 5^NULL as result3, + NULL^NULL as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 6 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + it(title="Arithmetic modulus with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + // Note % and mod() have different implemntations + var actual = queryExecute( + "SELECT 21%11 AS result, + NULL%5 as result2, + 5%NULL as result3, + NULL%NULL as result4, + mod( 21, 11 ) AS result5, + mod( NULL, 5 ) as result6, + mod( 5, NULL ) as result7, + mod( NULL, NULL ) as result8 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 10 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + expect( actual.result5 ).toBe( 10 ); + expect( actual.result6 ).toBeNull(); + expect( actual.result7 ).toBeNull(); + expect( actual.result8 ).toBeNull(); + }); + + it(title="Arithmetic exponent with NULL in QoQ", body=function( currentSpec ){ + qry = QueryNew('foo','integer',[[40]]); + var actual = queryExecute( + "SELECT power( 4, 2 ) AS result, + power( NULL, 5 ) as result2, + power( 5, NULL ) as result3, + power( NULL, NULL ) as result4 + FROM qry", + [], + {dbtype="query"} ); + expect( actual.result ).toBe( 16 ); + expect( actual.result2 ).toBeNull(); + expect( actual.result3 ).toBeNull(); + expect( actual.result4 ).toBeNull(); + }); + + }); + + } + + +} + diff --git a/src/test/java/external/specs/QoQNullGroupingTest.cfc b/src/test/java/external/specs/QoQNullGroupingTest.cfc new file mode 100644 index 000000000..6c5468825 --- /dev/null +++ b/src/test/java/external/specs/QoQNullGroupingTest.cfc @@ -0,0 +1,78 @@ +/** +* Copied from test\tickets\LDEV3640.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults , testBox ) { + + describe( 'QofQ nulls' , function(){ + + it( 'Stay null when unioned' , function() { + var qs_result = queryNew("col" , "int", [1]); + ``` + + select null as test + from qs_result + union + select null as test + from qs_result + + ``` + + expect( isNull( qs_result.test) ).toBeTrue(); + }); + + it( 'Stay null when grouped' , function() { + var qs_result = queryNew("col,col2" , "string,string", [ + ['test',nullValue()], + ['foo',nullValue()], + ['test',nullValue()] + ]); + ``` + + select col,col2,isnull(col2,42) as test + from qs_result + group by col,col2 + + ``` + + expect( isNull( qs_result.col2 ) ).toBeTrue(); + expect( qs_result.test ).toBe( 42 ); + + }); + + it( 'Stay null when aggregated' , function() { + + var qs_result = queryNew("dnum_auto,amount_local" , "integer,integer", [[10,10],[20,20]]); + ``` + + select sum(amount_local) as amount_local + from qs_result + where dnum_auto = 1000 + + ``` + + expect( testquery.recordCount ).toBe( 1 ); + expect( testquery.amount_local ).toBeNull(); + expect( isNull( testquery.amount_local ) ).toBeTrue(); + + ``` + + select sum(amount_local) as amount_local + from testquery + + ``` + + expect( testquery2.recordCount ).toBe( 1 ); + expect( testquery2.amount_local ).toBeNull(); + expect( isNull( testquery2.amount_local ) ).toBeTrue(); + }); + + }); + + } + + + +} + diff --git a/src/test/java/external/specs/QoQSpecialCharsTest.cfc b/src/test/java/external/specs/QoQSpecialCharsTest.cfc new file mode 100644 index 000000000..fc4f7eb51 --- /dev/null +++ b/src/test/java/external/specs/QoQSpecialCharsTest.cfc @@ -0,0 +1,125 @@ +/** +* Copied from test\tickets\LDEV4593.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + + function run( testResults, testBox ){ + // all your suites go here. + describe( "test qoq support with $", function(){ + + it( "test native qoq column names ", function(){ + + var q = querynew( "id$lucee" ); + queryAddRow( q ); + querySetCell( q, "id$lucee", 3 ); + query name="local.r" dbtype="query" { + echo( "select id$lucee from q" ); + } + expect( r[ "id$lucee" ] ).toBe( 3 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + + xit( "test native qoq with $ in column and leading a table name", function(){ + + var $q = querynew( "id$lucee" ); + queryAddRow( $q ); + querySetCell( $q, "id$lucee", 3 ); + query name="local.r" dbtype="query" { + echo( "select id$lucee from $q" ); + } + expect( r[ "id$lucee" ] ).toBe( 3 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + it( "test native qoq with $ in column name and $ inside a table name", function(){ + + var q$1 = querynew( "id$lucee" ); + queryAddRow( q$1 ); + querySetCell( q$1, "id$lucee", 3 ); + query name="local.r" dbtype="query" { + echo( "select id$lucee from q$1" ); + } + expect( r[ "id$lucee" ] ).toBe( 3 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + xit( "test native qoq with $ in column name and _ leading a table name", function(){ + + var _q = querynew( "id$lucee" ); + queryAddRow( _q ); + querySetCell( _q, "id$lucee", 3 ); + query name="local.r" dbtype="query" { + echo( "select id$lucee from _q" ); + } + expect( r[ "id$lucee" ] ).toBe( 3 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + + it( "test hsqldb qoq with $ in column names ", function(){ + + var q = querynew( "id$lucee" ); + queryAddRow( q ); + querySetCell( q, "id$lucee", 4 ); + query name="local.r" dbtype="query" { + echo( "select q1.id$lucee from q q1, q q2 where q1.id$lucee = q2.id$lucee" ); // join to force hsqldb + } + + expect( r[ "id$lucee" ] ).toBe( 4 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + xit( "test hsqldb qoq with $ in column name and $ leading table name", function(){ + + var $q = querynew( "id$lucee" ); + queryAddRow( $q ); + querySetCell( $q, "id$lucee", 4 ); + query name="local.r" dbtype="query" { + echo( "select q1.id$lucee from $q q1, $q q2 where q1.id$lucee = q2.id$lucee" ); // join to force hsqldb + } + + expect( r[ "id$lucee" ] ).toBe( 4 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + xit( "test hsqldb qoq with $ in column name and _ leading table name", function(){ + + var _q = querynew( "id$lucee" ); + queryAddRow( _q ); + querySetCell( _q, "id$lucee", 4 ); + query name="local.r" dbtype="query" { + echo( "select q1.id$lucee from _q q1, _q q2 where q1.id$lucee = q2.id$lucee" ); // join to force hsqldb + } + + expect( r[ "id$lucee" ] ).toBe( 4 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + it( "test hsqldb qoq with $ in column name and $ in table name", function(){ + + var q$1 = querynew( "id$lucee" ); + queryAddRow( q$1 ); + querySetCell( q$1, "id$lucee", 4 ); + query name="local.r" dbtype="query" { + echo( "select q1.id$lucee from q$1 q1, q$1 q2 where q1.id$lucee = q2.id$lucee" ); // join to force hsqldb + } + + expect( r[ "id$lucee" ] ).toBe( 4 ); + expect( r.recordcount ).toBe( 1 ); + + }); + + } ); + } + + +} + diff --git a/src/test/java/external/specs/QoQUnionParamTest.cfc b/src/test/java/external/specs/QoQUnionParamTest.cfc new file mode 100644 index 000000000..b9c515633 --- /dev/null +++ b/src/test/java/external/specs/QoQUnionParamTest.cfc @@ -0,0 +1,45 @@ +/** +* Copied from test\tickets\_LDEV3615.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + function run( testResults , testBox ) { + describe( "test case for LDEV-2128", function() { + it(title = "Query of Queries UNION returns incorrect results with cfqueryparam", body = function( currentSpec ) { + var testQuery = QueryNew('abcd'); + QueryAddRow(testQuery); + + local.qry = queryExecute( " + select :paramValue as Number, 'a' as Letter from testQuery + union + select :paramValue as Number, 'b' as Letter from testQuery + ", + {paramValue:{type:'cf_sql_integer',value:'1'}}, + {dbtype="query"} + ); + + expect(local.qry.Number[1]).toBe(1); + expect(local.qry.Number[2]).toBe(1); + expect(local.qry.letter[1]).toBe('a'); + expect(local.qry.letter[2]).toBe('b'); + }); + + it(title = "Query of Queries UNION returns correct results without cfqueryparam", body = function( currentSpec ) { + var testQuery = QueryNew('abcd'); + QueryAddRow(testQuery); + query name="local.qry" dbtype="query" { + echo(" + select 1 as Number, 'a' as Letter from testQuery + union + select 1 as Number, 'b' as Letter from testQuery + + "); + } + expect(local.qry.Number[1]).toBe(1); + expect(local.qry.Number[2]).toBe(1); + expect(local.qry.letter[1]).toBe('a'); + expect(local.qry.letter[2]).toBe('b'); + }); + }); + } +} + diff --git a/src/test/java/external/specs/QoQValidationChecksTest.cfc b/src/test/java/external/specs/QoQValidationChecksTest.cfc new file mode 100644 index 000000000..9e08c68e3 --- /dev/null +++ b/src/test/java/external/specs/QoQValidationChecksTest.cfc @@ -0,0 +1,136 @@ +/** +* Copied from test\tickets\LDEV3878.cfc in Lucee +*/ +component extends="testbox.system.BaseSpec"{ + q = queryNew("id","numeric",[[1]]); + + function run( testResults , testBox ) { + describe( title="LDEV-3878 QoQ should not fall back to HSQLDB and throw meaningful errors", body=function() { + + aroundEach(function( spec, suite ){ + try { + arguments.spec.body(); + } catch( any e ) { + return; + } + fail( 'Test did not throw exception as expected' ); + }); + + it(title="should throw for mod zero ", body = function( currentSpec ) { + queryExecute( " + select 1%0 + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for divide by zero ", body = function( currentSpec ) { + queryExecute( " + select 1/0 + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for too many count() columns ", body = function( currentSpec ) { + queryExecute( " + select count( id, id ) + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for too many count distinct columns ", body = function( currentSpec ) { + queryExecute( " + select count( distinct id, id ) + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for cast missing the type ", body = function( currentSpec ) { + queryExecute( " + select cast( id ) + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for Union with mismatched column counts ", body = function( currentSpec ) { + queryExecute( " + select id, '' + from q + UNION + select id + from q + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for union with invalid order by", body = function( currentSpec ) { + queryExecute( " + select id + from q + UNION + select id + from q + ORDER BY 'TEST' + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for invalid order by ordinal position ", body = function( currentSpec ) { + queryExecute( " + select id + from q + ORDER BY 5 + ", + [], + {dbtype:"query"} + ); + }); + + it(title="should throw for positional param value with wrong type", body = function( currentSpec ) { + queryExecute( " + select id + from q + where id= ? + ", + [ + {type="integer", value=""} + ], + {dbtype:"query"} + ); + }); + + it(title="should throw for named param value with wrong type", body = function( currentSpec ) { + queryExecute(" + select id + from q + where id= :id + ", + { + 'id': {type="integer", value=""} + }, + {dbtype:"query"} + ); + }); + + }); + } +} + diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 850d6648d..84fbc1463 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import ortus.boxlang.compiler.parser.ParsingResult; @@ -561,23 +562,23 @@ public void testNullAggregate() { } @Test + @Disabled public void testsdf() { instance.executeSource( """ - a = queryNew("a","varchar"); - b = queryNew("a","varchar", [['1'],['2'],['2']]); - - actual = QueryExecute( - sql = " - select a - from a - union select a - from b", - options = { dbtype: 'query' } + q = queryNew("id","numeric",[[1]]); + + queryExecute(" + select id + from q + where id= :id + ", + { + 'id': {type="integer", value=""} + }, + {dbtype:"query"} ); - - println( actual ) - """, + """, context ); } From 8d8ba0d0640f0c7fe7e94e75a244422478354d7f Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 20 Dec 2024 21:51:25 -0600 Subject: [PATCH 045/161] Use SLL mode BL-823 --- src/main/antlr/SQLGrammar.g4 | 489 +----------------- src/main/antlr/SQLLexer.g4 | 121 +---- .../compiler/parser/SQLLexerCustom.java | 63 ++- .../boxlang/compiler/parser/SQLParser.java | 69 ++- .../compiler/toolchain/SQLVisitor.java | 3 +- 5 files changed, 138 insertions(+), 607 deletions(-) diff --git a/src/main/antlr/SQLGrammar.g4 b/src/main/antlr/SQLGrammar.g4 index 620664969..f7827ef45 100644 --- a/src/main/antlr/SQLGrammar.g4 +++ b/src/main/antlr/SQLGrammar.g4 @@ -17,92 +17,7 @@ sql_stmt_list: SCOL* sql_stmt (SCOL+ sql_stmt)* SCOL* ; -sql_stmt: (EXPLAIN_ (QUERY_ PLAN_)?)? ( - alter_table_stmt - | analyze_stmt - | attach_stmt - | begin_stmt - | commit_stmt - | create_index_stmt - | create_table_stmt - | create_trigger_stmt - | create_view_stmt - | create_virtual_table_stmt - | delete_stmt - | delete_stmt_limited - | detach_stmt - | drop_stmt - | insert_stmt - | pragma_stmt - | reindex_stmt - | release_stmt - | rollback_stmt - | savepoint_stmt - | select_stmt - | update_stmt - | update_stmt_limited - | vacuum_stmt - ) -; - -alter_table_stmt: - ALTER_ TABLE_ (schema_name DOT)? table_name ( - RENAME_ ( - TO_ new_table_name = table_name - | COLUMN_? old_column_name = column_name TO_ new_column_name = column_name - ) - | ADD_ COLUMN_? column_def - | DROP_ COLUMN_? column_name - ) -; - -analyze_stmt: - ANALYZE_ (schema_name | (schema_name DOT)? table_or_index_name)? -; - -attach_stmt: - ATTACH_ DATABASE_? expr AS_ schema_name -; - -begin_stmt: - BEGIN_ (DEFERRED_ | IMMEDIATE_ | EXCLUSIVE_)? (TRANSACTION_ transaction_name?)? -; - -commit_stmt: (COMMIT_ | END_) TRANSACTION_? -; - -rollback_stmt: - ROLLBACK_ TRANSACTION_? (TO_ SAVEPOINT_? savepoint_name)? -; - -savepoint_stmt: - SAVEPOINT_ savepoint_name -; - -release_stmt: - RELEASE_ SAVEPOINT_? savepoint_name -; - -create_index_stmt: - CREATE_ UNIQUE_? INDEX_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? index_name ON_ table_name OPEN_PAR indexed_column ( - COMMA indexed_column - )* CLOSE_PAR (WHERE_ expr)? -; - -indexed_column: (column_name | expr) (COLLATE_ collation_name)? asc_desc? -; - -create_table_stmt: - CREATE_ (TEMP_ | TEMPORARY_)? TABLE_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? table_name ( - OPEN_PAR column_def (COMMA column_def)*? (COMMA table_constraint)* CLOSE_PAR ( - WITHOUT_ row_ROW_ID = IDENTIFIER - )? - | AS_ select_stmt - ) -; - -column_def: - column_name type_name? column_constraint* +sql_stmt: (select_stmt) ; type_name: @@ -112,73 +27,9 @@ type_name: )? ; -column_constraint: (CONSTRAINT_ name)? ( - (PRIMARY_ KEY_ asc_desc? conflict_clause? AUTOINCREMENT_?) - | (NOT_? NULL_ | UNIQUE_) conflict_clause? - | CHECK_ OPEN_PAR expr CLOSE_PAR - | DEFAULT_ (signed_number | literal_value | OPEN_PAR expr CLOSE_PAR) - | COLLATE_ collation_name - | foreign_key_clause - | (GENERATED_ ALWAYS_)? AS_ OPEN_PAR expr CLOSE_PAR (STORED_ | VIRTUAL_)? - ) -; - signed_number: (PLUS | MINUS)? NUMERIC_LITERAL ; -table_constraint: (CONSTRAINT_ name)? ( - (PRIMARY_ KEY_ | UNIQUE_) OPEN_PAR indexed_column (COMMA indexed_column)* CLOSE_PAR conflict_clause? - | CHECK_ OPEN_PAR expr CLOSE_PAR - | FOREIGN_ KEY_ OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR foreign_key_clause - ) -; - -foreign_key_clause: - REFERENCES_ foreign_table (OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR)? ( - ON_ (DELETE_ | UPDATE_) ( - SET_ (NULL_ | DEFAULT_) - | CASCADE_ - | RESTRICT_ - | NO_ ACTION_ - ) - | MATCH_ name - )* (NOT_? DEFERRABLE_ (INITIALLY_ (DEFERRED_ | IMMEDIATE_))?)? -; - -conflict_clause: - ON_ CONFLICT_ (ROLLBACK_ | ABORT_ | FAIL_ | IGNORE_ | REPLACE_) -; - -create_trigger_stmt: - CREATE_ (TEMP_ | TEMPORARY_)? TRIGGER_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? trigger_name ( - BEFORE_ - | AFTER_ - | INSTEAD_ OF_ - )? (DELETE_ | INSERT_ | UPDATE_ (OF_ column_name ( COMMA column_name)*)?) ON_ table_name ( - FOR_ EACH_ ROW_ - )? (WHEN_ expr)? BEGIN_ ( - (update_stmt | insert_stmt | delete_stmt | select_stmt) SCOL - )+ END_ -; - -create_view_stmt: - CREATE_ (TEMP_ | TEMPORARY_)? VIEW_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? view_name ( - OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR - )? AS_ select_stmt -; - -create_virtual_table_stmt: - CREATE_ VIRTUAL_ TABLE_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? table_name USING_ module_name ( - OPEN_PAR module_argument (COMMA module_argument)* CLOSE_PAR - )? -; - -with_clause: - WITH_ RECURSIVE_? cte_table_name AS_ OPEN_PAR select_stmt CLOSE_PAR ( - COMMA cte_table_name AS_ OPEN_PAR select_stmt CLOSE_PAR - )* -; - cte_table_name: table_name (OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR)? ; @@ -191,26 +42,6 @@ common_table_expression: table_name (OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR)? AS_ OPEN_PAR select_stmt CLOSE_PAR ; -delete_stmt: - with_clause? DELETE_ FROM_ qualified_table_name (WHERE_ expr)? returning_clause? -; - -delete_stmt_limited: - with_clause? DELETE_ FROM_ qualified_table_name (WHERE_ expr)? returning_clause? ( - order_by_stmt? limit_stmt - )? -; - -detach_stmt: - DETACH_ DATABASE_? schema_name -; - -drop_stmt: - DROP_ object = (INDEX_ | TABLE_ | TRIGGER_ | VIEW_) (IF_ EXISTS_)? ( - schema_name DOT - )? any_name -; - predicate: expr (LT | LT_EQ | GT | GT_EQ) expr | expr (ASSIGN | EQ | NOT_EQ1 | NOT_EQ2 | IS_ NOT_ | IS_ | LIKE_) expr @@ -236,9 +67,9 @@ expr: // math or maybe concat operators | expr (PLUS | MINUS) expr // Special handling of cast to allow cast( foo as number) - | CAST_ OPEN_PAR expr AS_ (name | STRING_LITERAL) CLOSE_PAR + | CAST_ OPEN_PAR expr AS_ (IDENTIFIER | STRING_LITERAL) CLOSE_PAR // special handling of convert to allow convert( foo, number ) or convert( foo, 'number' ) - | CONVERT_ OPEN_PAR expr COMMA (name | STRING_LITERAL) CLOSE_PAR + | CONVERT_ OPEN_PAR expr COMMA (IDENTIFIER | STRING_LITERAL) CLOSE_PAR | function_name OPEN_PAR ((DISTINCT_? ALL_? expr ( COMMA expr)*) | STAR)? CLOSE_PAR // filter_clause? over_clause? | OPEN_PAR expr CLOSE_PAR | case_expr @@ -252,76 +83,24 @@ case_when_then: WHEN_ (when_expr = expr | when_predicate = predicate) THEN_ then_expr = expr ; -raise_function: - RAISE_ OPEN_PAR (IGNORE_ | (ROLLBACK_ | ABORT_ | FAIL_) COMMA error_message) CLOSE_PAR -; - literal_value: NUMERIC_LITERAL | STRING_LITERAL - // | BLOB_LITERAL | NULL_ | TRUE_ | FALSE_ - // | CURRENT_TIME_ - // | CURRENT_DATE_ - // | CURRENT_TIMESTAMP_ ; value_row: OPEN_PAR expr (COMMA expr)* CLOSE_PAR ; -values_clause: - VALUES_ value_row (COMMA value_row)* -; - -insert_stmt: - with_clause? ( - INSERT_ - | REPLACE_ - | INSERT_ OR_ ( REPLACE_ | ROLLBACK_ | ABORT_ | FAIL_ | IGNORE_) - ) INTO_ (schema_name DOT)? table_name (AS_ table_alias)? ( - OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR - )? (( ( values_clause | select_stmt) upsert_clause?) | DEFAULT_ VALUES_) returning_clause? -; - -returning_clause: - RETURNING_ result_column (COMMA result_column)* -; - -upsert_clause: - ON_ CONFLICT_ ( - OPEN_PAR indexed_column (COMMA indexed_column)* CLOSE_PAR (WHERE_ expr)? - )? DO_ ( - NOTHING_ - | UPDATE_ SET_ ( - (column_name | column_name_list) ASSIGN expr ( - COMMA (column_name | column_name_list) ASSIGN expr - )* (WHERE_ expr)? - ) - ) -; - -pragma_stmt: - PRAGMA_ (schema_name DOT)? pragma_name ( - ASSIGN pragma_value - | OPEN_PAR pragma_value CLOSE_PAR - )? -; - pragma_value: signed_number | name | STRING_LITERAL ; -reindex_stmt: - REINDEX_ (collation_name | (schema_name DOT)? (table_name | index_name))? -; - -//select_stmt: common_table_stmt? select_core (compound_operator select_core)* order_by_stmt? limit_stmt?; - select_stmt: select_core (union)* order_by_stmt? limit_stmt? ; @@ -346,8 +125,6 @@ select_core: HAVING_ havingExpr = predicate )? )? limit_stmt? - //(WINDOW_ window_name AS_ window_defn ( COMMA window_name AS_ window_defn)*)? - // | values_clause ; top: @@ -358,16 +135,6 @@ factored_select_stmt: select_stmt ; -simple_select_stmt: - common_table_stmt? select_core order_by_stmt? limit_stmt? -; - -compound_select_stmt: - common_table_stmt? select_core ( - (UNION_ ALL_? | INTERSECT_ | EXCEPT_) select_core - )+ order_by_stmt? limit_stmt? -; - table: (schema_name DOT)? table_name (AS_? table_alias)? ; @@ -392,102 +159,17 @@ result_column: ; join_operator: - // COMMA - // | NATURAL_? ((LEFT_ | RIGHT_ | FULL_) OUTER_? | INNER_ | CROSS_)? JOIN_ ((LEFT_ | RIGHT_ | FULL_) OUTER_? | INNER_ | CROSS_)? JOIN_ ; join_constraint: ON_ predicate - // | USING_ OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR -; - -compound_operator: - UNION_ ALL_? - | INTERSECT_ - | EXCEPT_ -; - -update_stmt: - with_clause? UPDATE_ (OR_ (ROLLBACK_ | ABORT_ | REPLACE_ | FAIL_ | IGNORE_))? qualified_table_name SET_ ( - column_name - | column_name_list - ) ASSIGN expr (COMMA (column_name | column_name_list) ASSIGN expr)* ( - FROM_ (table_or_subquery (COMMA table_or_subquery)* | join_clause) - )? (WHERE_ expr)? returning_clause? -; - -column_name_list: - OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR -; - -update_stmt_limited: - with_clause? UPDATE_ (OR_ (ROLLBACK_ | ABORT_ | REPLACE_ | FAIL_ | IGNORE_))? qualified_table_name SET_ ( - column_name - | column_name_list - ) ASSIGN expr (COMMA (column_name | column_name_list) ASSIGN expr)* (WHERE_ expr)? returning_clause? ( - order_by_stmt? limit_stmt - )? -; - -qualified_table_name: (schema_name DOT)? table_name (AS_ alias)? ( - INDEXED_ BY_ index_name - | NOT_ INDEXED_ - )? -; - -vacuum_stmt: - VACUUM_ schema_name? (INTO_ filename)? -; - -filter_clause: - FILTER_ OPEN_PAR WHERE_ expr CLOSE_PAR -; - -window_defn: - OPEN_PAR base_window_name? (PARTITION_ BY_ expr (COMMA expr)*)? ( - ORDER_ BY_ ordering_term (COMMA ordering_term)* - ) frame_spec? CLOSE_PAR -; - -over_clause: - OVER_ ( - window_name - | OPEN_PAR base_window_name? (PARTITION_ BY_ expr (COMMA expr)*)? ( - ORDER_ BY_ ordering_term (COMMA ordering_term)* - )? frame_spec? CLOSE_PAR - ) -; - -frame_spec: - frame_clause (EXCLUDE_ ( NO_ OTHERS_ | CURRENT_ ROW_ | GROUP_ | TIES_))? -; - -frame_clause: (RANGE_ | ROWS_ | GROUPS_) ( - frame_single - | BETWEEN_ frame_left AND_ frame_right - ) ; simple_function_invocation: simple_func OPEN_PAR (expr (COMMA expr)* | STAR) CLOSE_PAR ; -aggregate_function_invocation: - aggregate_func OPEN_PAR (DISTINCT_? expr (COMMA expr)* | STAR)? CLOSE_PAR filter_clause? -; - -window_function_invocation: - window_function OPEN_PAR (expr (COMMA expr)* | STAR)? CLOSE_PAR filter_clause? OVER_ ( - window_defn - | window_name - ) -; - -common_table_stmt: //additional structures - WITH_ RECURSIVE_? common_table_expression (COMMA common_table_expression)* -; - order_by_stmt: ORDER_ BY_ ordering_term (COMMA ordering_term)* ; @@ -507,38 +189,6 @@ asc_desc: | DESC_ ; -frame_left: - expr PRECEDING_ - | expr FOLLOWING_ - | CURRENT_ ROW_ - | UNBOUNDED_ PRECEDING_ -; - -frame_right: - expr PRECEDING_ - | expr FOLLOWING_ - | CURRENT_ ROW_ - | UNBOUNDED_ FOLLOWING_ -; - -frame_single: - expr PRECEDING_ - | UNBOUNDED_ PRECEDING_ - | CURRENT_ ROW_ -; - -// unknown - -window_function: (FIRST_VALUE_ | LAST_VALUE_) OPEN_PAR expr CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc - frame_clause? CLOSE_PAR - | (CUME_DIST_ | PERCENT_RANK_) OPEN_PAR CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr? CLOSE_PAR - | (DENSE_RANK_ | RANK_ | ROW_NUMBER_) OPEN_PAR CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc CLOSE_PAR - | (LAG_ | LEAD_) OPEN_PAR expr offset? default_value? CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc CLOSE_PAR - | NTH_VALUE_ OPEN_PAR expr COMMA signed_number CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc frame_clause? - CLOSE_PAR - | NTILE_ OPEN_PAR expr CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc CLOSE_PAR -; - offset: COMMA signed_number ; @@ -547,10 +197,6 @@ default_value: COMMA signed_number ; -partition_by: - PARTITION_ BY_ expr+ -; - order_by_expr: ORDER_ BY_ expr+ ; @@ -584,173 +230,52 @@ error_message: STRING_LITERAL ; -module_argument: // TODO check what exactly is permitted here - expr - | column_def -; - column_alias: IDENTIFIER | STRING_LITERAL ; keyword: - ABORT_ - | ACTION_ - | ADD_ - | AFTER_ - | ALL_ + ALL_ | ALTER_ - | ANALYZE_ | AND_ | AS_ | ASC_ - | ATTACH_ - | AUTOINCREMENT_ - | BEFORE_ | BEGIN_ | BETWEEN_ | BY_ - | CASCADE_ | CASE_ | CAST_ | CONVERT_ - | CHECK_ - | COLLATE_ - | COLUMN_ - | COMMIT_ - | CONFLICT_ - | CONSTRAINT_ - | CREATE_ | CROSS_ - | CURRENT_DATE_ - | CURRENT_TIME_ - | CURRENT_TIMESTAMP_ - | DATABASE_ - | DEFAULT_ - | DEFERRABLE_ - | DEFERRED_ - | DELETE_ | DESC_ - | DETACH_ | DISTINCT_ - | DROP_ - | EACH_ | ELSE_ | END_ | ESCAPE_ - | EXCEPT_ - | EXCLUSIVE_ - | EXISTS_ - | EXPLAIN_ - | FAIL_ - | FOR_ - | FOREIGN_ | FROM_ | FULL_ - | GLOB_ | GROUP_ | HAVING_ - | IF_ - | IGNORE_ - | IMMEDIATE_ | IN_ - | INDEX_ - | INDEXED_ - | INITIALLY_ | INNER_ - | INSERT_ - | INSTEAD_ - | INTERSECT_ - | INTO_ | IS_ - | ISNULL_ | JOIN_ - | KEY_ | LEFT_ | LIKE_ | LIMIT_ - | MATCH_ - | NATURAL_ - | NO_ | NOT_ - | NOTNULL_ | NULL_ - | OF_ - | OFFSET_ | ON_ | OR_ | ORDER_ | OUTER_ - | PLAN_ - | PRAGMA_ - | PRIMARY_ - | QUERY_ - | RAISE_ - | RECURSIVE_ - | REFERENCES_ - | REGEXP_ - | REINDEX_ - | RELEASE_ - | RENAME_ - | REPLACE_ - | RESTRICT_ - | RIGHT_ - | ROLLBACK_ - | ROW_ - | ROWS_ - | SAVEPOINT_ | SELECT_ - | SET_ - | TABLE_ - | TEMP_ - | TEMPORARY_ | THEN_ - | TO_ - | TRANSACTION_ - | TRIGGER_ - | UNION_ - | UNIQUE_ - | UPDATE_ - | USING_ - | VACUUM_ - | VALUES_ - | VIEW_ - | VIRTUAL_ | WHEN_ | WHERE_ - | WITH_ - | WITHOUT_ - | FIRST_VALUE_ - | OVER_ - | PARTITION_ - | RANGE_ - | PRECEDING_ - | UNBOUNDED_ - | CURRENT_ - | FOLLOWING_ - | CUME_DIST_ - | DENSE_RANK_ - | LAG_ - | LAST_VALUE_ - | LEAD_ - | NTH_VALUE_ - | NTILE_ - | PERCENT_RANK_ - | RANK_ - | ROW_NUMBER_ - | GENERATED_ - | ALWAYS_ - | STORED_ | TRUE_ | FALSE_ - | WINDOW_ - | NULLS_ - | FIRST_ - | LAST_ - | FILTER_ - | GROUPS_ - | EXCLUDE_ ; // TODO: check all names below @@ -760,7 +285,7 @@ name: ; function_name: - any_name + FUNCTION_NAME ; schema_name: @@ -824,7 +349,7 @@ window_name: ; alias: - any_name + IDENTIFIER ; filename: @@ -850,6 +375,4 @@ table_function_name: any_name: IDENTIFIER | keyword - // | STRING_LITERAL - //| OPEN_PAR any_name CLOSE_PAR ; \ No newline at end of file diff --git a/src/main/antlr/SQLLexer.g4 b/src/main/antlr/SQLLexer.g4 index 291b77462..6f88837a2 100644 --- a/src/main/antlr/SQLLexer.g4 +++ b/src/main/antlr/SQLLexer.g4 @@ -39,168 +39,51 @@ NOT_EQ1: '!='; BANG: '!'; NOT_EQ2: '<>'; -ABORT_: 'ABORT'; -ACTION_: 'ACTION'; -ADD_: 'ADD'; -AFTER_: 'AFTER'; ALL_: 'ALL'; ALTER_: 'ALTER'; -ANALYZE_: 'ANALYZE'; AND_: 'AND'; AS_: 'AS'; ASC_: 'ASC'; -ATTACH_: 'ATTACH'; -AUTOINCREMENT_: 'AUTOINCREMENT'; -BEFORE_: 'BEFORE'; BEGIN_: 'BEGIN'; BETWEEN_: 'BETWEEN'; BY_: 'BY'; -CASCADE_: 'CASCADE'; CASE_: 'CASE'; CAST_: 'CAST'; CONVERT_: 'CONVERT'; -CHECK_: 'CHECK'; -COLLATE_: 'COLLATE'; -COLUMN_: 'COLUMN'; -COMMIT_: 'COMMIT'; -CONFLICT_: 'CONFLICT'; -CONSTRAINT_: 'CONSTRAINT'; -CREATE_: 'CREATE'; CROSS_: 'CROSS'; -CURRENT_DATE_: 'CURRENT_DATE'; -CURRENT_TIME_: 'CURRENT_TIME'; -CURRENT_TIMESTAMP_: 'CURRENT_TIMESTAMP'; -DATABASE_: 'DATABASE'; -DEFAULT_: 'DEFAULT'; -DEFERRABLE_: 'DEFERRABLE'; -DEFERRED_: 'DEFERRED'; -DELETE_: 'DELETE'; DESC_: 'DESC'; -DETACH_: 'DETACH'; DISTINCT_: 'DISTINCT'; -DROP_: 'DROP'; -EACH_: 'EACH'; ELSE_: 'ELSE'; END_: 'END'; ESCAPE_: 'ESCAPE'; -EXCEPT_: 'EXCEPT'; -EXCLUSIVE_: 'EXCLUSIVE'; -EXISTS_: 'EXISTS'; -EXPLAIN_: 'EXPLAIN'; -FAIL_: 'FAIL'; -FOR_: 'FOR'; -FOREIGN_: 'FOREIGN'; FROM_: 'FROM'; FULL_: 'FULL'; -GLOB_: 'GLOB'; GROUP_: 'GROUP'; HAVING_: 'HAVING'; -IF_: 'IF'; -IGNORE_: 'IGNORE'; -IMMEDIATE_: 'IMMEDIATE'; IN_: 'IN'; -INDEX_: 'INDEX'; -INDEXED_: 'INDEXED'; -INITIALLY_: 'INITIALLY'; INNER_: 'INNER'; -INSERT_: 'INSERT'; -INSTEAD_: 'INSTEAD'; -INTERSECT_: 'INTERSECT'; -INTO_: 'INTO'; IS_: 'IS'; -ISNULL_: 'ISNULL'; JOIN_: 'JOIN'; -KEY_: 'KEY'; LEFT_: 'LEFT'; LIKE_: 'LIKE'; LIMIT_: 'LIMIT'; -MATCH_: 'MATCH'; -NATURAL_: 'NATURAL'; -NO_: 'NO'; NOT_: 'NOT'; -NOTNULL_: 'NOTNULL'; NULL_: 'NULL'; -OF_: 'OF'; -OFFSET_: 'OFFSET'; ON_: 'ON'; OR_: 'OR'; ORDER_: 'ORDER'; OUTER_: 'OUTER'; -PLAN_: 'PLAN'; -PRAGMA_: 'PRAGMA'; -PRIMARY_: 'PRIMARY'; -QUERY_: 'QUERY'; -RAISE_: 'RAISE'; -RECURSIVE_: 'RECURSIVE'; -REFERENCES_: 'REFERENCES'; -REGEXP_: 'REGEXP'; -REINDEX_: 'REINDEX'; -RELEASE_: 'RELEASE'; -RENAME_: 'RENAME'; -REPLACE_: 'REPLACE'; -RESTRICT_: 'RESTRICT'; -RETURNING_: 'RETURNING'; RIGHT_: 'RIGHT'; -ROLLBACK_: 'ROLLBACK'; -ROW_: 'ROW'; -ROWS_: 'ROWS'; -SAVEPOINT_: 'SAVEPOINT'; SELECT_: 'SELECT'; -SET_: 'SET'; -TABLE_: 'TABLE'; -TEMP_: 'TEMP'; -TEMPORARY_: 'TEMPORARY'; THEN_: 'THEN'; -TO_: 'TO'; TOP: 'TOP'; -TRANSACTION_: 'TRANSACTION'; -TRIGGER_: 'TRIGGER'; UNION_: 'UNION'; -UNIQUE_: 'UNIQUE'; -UPDATE_: 'UPDATE'; -USING_: 'USING'; -VACUUM_: 'VACUUM'; -VALUES_: 'VALUES'; -VIEW_: 'VIEW'; -VIRTUAL_: 'VIRTUAL'; WHEN_: 'WHEN'; WHERE_: 'WHERE'; -WITH_: 'WITH'; -WITHOUT_: 'WITHOUT'; -FIRST_VALUE_: 'FIRST_VALUE'; -OVER_: 'OVER'; -PARTITION_: 'PARTITION'; -RANGE_: 'RANGE'; -PRECEDING_: 'PRECEDING'; -UNBOUNDED_: 'UNBOUNDED'; -CURRENT_: 'CURRENT'; -FOLLOWING_: 'FOLLOWING'; -CUME_DIST_: 'CUME_DIST'; -DENSE_RANK_: 'DENSE_RANK'; -LAG_: 'LAG'; -LAST_VALUE_: 'LAST_VALUE'; -LEAD_: 'LEAD'; -NTH_VALUE_: 'NTH_VALUE'; -NTILE_: 'NTILE'; -PERCENT_RANK_: 'PERCENT_RANK'; -RANK_: 'RANK'; -ROW_NUMBER_: 'ROW_NUMBER'; -GENERATED_: 'GENERATED'; -ALWAYS_: 'ALWAYS'; -STORED_: 'STORED'; TRUE_: 'TRUE'; FALSE_: 'FALSE'; -WINDOW_: 'WINDOW'; -NULLS_: 'NULLS'; -FIRST_: 'FIRST'; -LAST_: 'LAST'; -FILTER_: 'FILTER'; -GROUPS_: 'GROUPS'; -EXCLUDE_: 'EXCLUDE'; -TIES_: 'TIES'; -OTHERS_: 'OTHERS'; -DO_: 'DO'; -NOTHING_: 'NOTHING'; + +FUNCTION_NAME: . {false}?; IDENTIFIER: '"' (~'"' | '""')* '"' diff --git a/src/main/java/ortus/boxlang/compiler/parser/SQLLexerCustom.java b/src/main/java/ortus/boxlang/compiler/parser/SQLLexerCustom.java index 6d23b25d0..14dacfb7f 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/SQLLexerCustom.java +++ b/src/main/java/ortus/boxlang/compiler/parser/SQLLexerCustom.java @@ -16,8 +16,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.Token; import ortus.boxlang.parser.antlr.SQLLexer; @@ -31,12 +33,32 @@ public class SQLLexerCustom extends SQLLexer { /** * The error listener */ - ErrorListener errorListener; + ErrorListener errorListener; /** * The parser */ - SQLParser parser; + SQLParser parser; + + /** + * A reference to the last token + */ + Token lastToken = null; + + /** + * A flag to check if we are in a cast + */ + int inCast = 0; + + /** + * These tokens are not function names. cast and convert have a special rule to match them, so they don't use the FUNCTION_NAME token type + */ + private static final Set notFunctionNames = Set.of( NOT_, AND_, WHERE_, HAVING_, FROM_, IN_, ON_, CAST_, CONVERT_ ); + + /** + * ASCII Character code for left parenthesis + */ + private int LPAREN_Char_Code = 40; /** * Constructor @@ -180,4 +202,41 @@ public List findPreviousTokenAndXSiblings( int type, int count ) { return results; } + public Token nextToken() { + Token nextToken = super.nextToken(); + + if ( lastToken == null ) { + return setLastToken( nextToken ); + } + + if ( nextToken.getType() == OPEN_PAR && lastToken.getType() == CAST_ || inCast > 0 ) { + inCast++; + } + + if ( nextToken.getType() == CLOSE_PAR && inCast > 0 ) { + inCast--; + } + + // Any token after AS is an identifier + if ( lastToken.getType() == SQLLexer.AS_ && nextToken.getType() != SQLLexer.SPACES && inCast == 0 ) { + ( ( CommonToken ) nextToken ).setType( IDENTIFIER ); + return setLastToken( nextToken ); + } + + // detect function calls and set the token type to FUNCTION_NAME + if ( getInputStream().LA( 1 ) == LPAREN_Char_Code && !notFunctionNames.contains( nextToken.getType() ) ) { + ( ( CommonToken ) nextToken ).setType( SQLLexer.FUNCTION_NAME ); + return setLastToken( nextToken ); + } + + return setLastToken( nextToken ); + } + + private Token setLastToken( Token token ) { + if ( token.getChannel() != HIDDEN ) { + lastToken = token; + } + return token; + } + } \ No newline at end of file diff --git a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java index 53fccc2f6..75927b6cd 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java +++ b/src/main/java/ortus/boxlang/compiler/parser/SQLParser.java @@ -17,12 +17,18 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.AmbiguityInfo; +import org.antlr.v4.runtime.atn.DecisionInfo; +import org.antlr.v4.runtime.atn.DecisionState; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.misc.Interval; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BOMInputStream; @@ -141,13 +147,19 @@ protected BoxNode parserFirstStage( InputStream stream, boolean classOrInterface SQLGrammar parser = new SQLGrammar( new CommonTokenStream( lexer ) ); // DEBUG: Will print a trace of all parser rules visited: - // boxParser.setTrace( true ); addErrorListeners( lexer, parser ); + // parser.setTrace( true ); parser.setErrorHandler( new BoxParserErrorStrategy() ); - // parser.getInterpreter().setPredictionMode( PredictionMode.SLL ); + parser.getInterpreter().setPredictionMode( PredictionMode.SLL ); + + // activating profiling + // parser.setProfile( true ); + ParserRuleContext parseTree = parser.parse(); + // profileParser( parser ); + // This must run FIRST before resetting the lexer validateParse( lexer ); @@ -179,6 +191,59 @@ protected BoxNode parserFirstStage( InputStream stream, boolean classOrInterface return rootNode; } + public void profileParser( SQLGrammar parser ) { + PrintStream out = System.out; + + out.printf( "%-35s", "rule" ); + out.printf( "%-15s", "time" ); + out.printf( "%-15s", "invocations" ); + out.printf( "%-15s", "lookahead" ); + out.printf( "%-15s", "lookahead(max)" ); + out.printf( "%-15s%n", "errors" ); + + for ( DecisionInfo decisionInfo : parser.getParseInfo().getDecisionInfo() ) { + DecisionState ds = parser.getATN().getDecisionState( decisionInfo.decision ); + String rule = parser.getRuleNames()[ ds.ruleIndex ]; + if ( decisionInfo.timeInPrediction > 0 ) { + out.printf( "%-35s", rule ); + out.printf( "%-15s", decisionInfo.timeInPrediction / 1_000_000D + "ms" ); + out.printf( "%-15s", decisionInfo.invocations ); + out.printf( "%-15s", decisionInfo.SLL_TotalLook ); + out.printf( "%-15s", decisionInfo.SLL_MaxLook ); + out.printf( "%-15s%n", decisionInfo.errors ); + + // out.printf( "%-15s", decisionInfo.ambiguities ); + for ( AmbiguityInfo ambiguity : decisionInfo.ambiguities ) { + out.println(); + + out.println( " **** Ambiguity ****" ); + DecisionState dsa = parser.getATN().getDecisionState( ambiguity.decision ); + String rulea = parser.getRuleNames()[ dsa.ruleIndex ]; + // out.println( " rule:" + rulea ); + // out.println( " fullCtx:" + ambiguity.fullCtx ); + + String ambiguousSubstring = ambiguity.input.getText( Interval.of( ambiguity.startIndex, ambiguity.stopIndex ) ); + out.println( " ambiguous text: [" + ambiguousSubstring + "]" ); + + out.println( " ambigAlts:" + ambiguity.ambigAlts ); + + // Iterate over the configurations and print only those that match the ambiguous alternatives + /* + * out.println( " Configurations:" ); + * for ( ATNConfig config : ambiguity.configs ) { + * if ( ambiguity.ambigAlts.get( config.alt ) ) { + * out.println( " State: " + config.state.stateNumber ); + * out.println( " Context: " + Arrays.toString( config.context.toStrings( parser, config.state.stateNumber ) ) ); + * out.println( " Alt: " + config.alt ); + * } + * } + */ + out.println(); + } + } + } + } + private void validateParse( SQLLexerCustom lexer ) { // Check if there are unconsumed tokens diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index 354ad22bd..ead79d298 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -477,7 +477,8 @@ public SQLExpression visitExpr( ExprContext ctx, SQLTable table, List j if ( ctx.STRING_LITERAL() != null ) { type = processStringLiteral( ctx.STRING_LITERAL() ); } else { - type = new SQLStringLiteral( unwrapBracket( ctx.name().getText() ), tools.getPosition( ctx.name() ), tools.getSourceText( ctx.name() ) ); + type = new SQLStringLiteral( unwrapBracket( ctx.IDENTIFIER().getText() ), tools.getPosition( ctx.IDENTIFIER() ), + ctx.IDENTIFIER().getText() ); } // validate the type here try { From 8bb1d0521acb940686912637ec3f51463f76d527 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sun, 22 Dec 2024 07:09:07 -0500 Subject: [PATCH 046/161] JDBC - Add list queryparam tests --- .../bifs/global/jdbc/QueryExecuteTest.java | 17 +++++++++++++++++ .../runtime/components/jdbc/QueryTest.java | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index a15205044..fb9290ce5 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -155,6 +155,23 @@ public void testArrayBindings() { assertEquals( "Developer", michael.get( "role" ) ); } + @Disabled( "Unimplemented" ) + @DisplayName( "It can execute a query with a list binding" ) + @Test + public void testListBindings() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id IN (:ids)", + { "ids" : { value: "77,1,42", list : true } } + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 3, query.size() ); + } + @DisplayName( "It can execute a query with struct bindings on the default datasource" ) @Test public void testStructBindings() { diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index c2b6ece16..eb6eff6e3 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -28,6 +28,7 @@ import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -157,6 +158,22 @@ public void testArrayBindings() { assertEquals( "Developer", michael.get( "role" ) ); } + @Disabled( "Unimplemented" ) + @DisplayName( "It can execute a query with a list queryparam" ) + @Test + public void testListBindings() { + getInstance().executeSource( + """ + + SELECT * FROM developers WHERE id IN () + + """, + context ); + assertThat( getVariables().get( result ) ).isInstanceOf( Query.class ); + ortus.boxlang.runtime.types.Query query = getVariables().getAsQuery( result ); + assertEquals( 3, query.size() ); + } + @DisplayName( "It can execute a query on a named datasource" ) @Test public void testNamedDataSource() { From 139d8d12874e51b6bec1b44d2c463c36c836b8e1 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sun, 22 Dec 2024 07:11:04 -0500 Subject: [PATCH 047/161] JDBC - Ensure we shutdown datasource connections when shutdown is called Don't just wipe the array, close the pools. :) --- .../java/ortus/boxlang/runtime/jdbc/ConnectionManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java b/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java index 016826067..70f58d327 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java @@ -595,6 +595,9 @@ public Map getCachedDatasources() { * Shutdown the ConnectionManager and release any resources. */ public void shutdown() { + this.datasources.forEach( ( key, datasource ) -> { + datasource.shutdown(); + } ); this.datasources.clear(); } From dea03f04bcc6e7284d49bf75ec2c654526a1a24f Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sun, 22 Dec 2024 07:14:14 -0500 Subject: [PATCH 048/161] JDBC - Fix list coercion for array lists in QueryParameter --- .../java/ortus/boxlang/runtime/jdbc/QueryParameter.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index 84b5cf5ee..f8f605190 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -24,6 +24,7 @@ import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.dynamic.casters.StructCaster; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.Struct; @@ -86,8 +87,12 @@ private QueryParameter( IStruct param ) { Object v = param.get( Key.value ); if ( this.isListParam ) { - v = ListUtil.asList( ( String ) v, ( String ) param.getOrDefault( Key.separator, "," ) ); - sqltype = "ARRAY"; + sqltype = "ARRAY"; + if ( v instanceof Array ) { + // do nothing? + } else { + v = ListUtil.asList( ( String ) v, ( String ) param.getOrDefault( Key.separator, "," ) ); + } } this.value = this.isNullParam ? null : v; From ce727d99594fb57a4bf10440bdc696fe621e522f Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sun, 22 Dec 2024 07:15:07 -0500 Subject: [PATCH 049/161] ListUtil - Coerce arrays to strings before attempting to collect into a string list Our ListUtil.asString() was missing array value support. --- src/main/java/ortus/boxlang/runtime/types/util/ListUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ortus/boxlang/runtime/types/util/ListUtil.java b/src/main/java/ortus/boxlang/runtime/types/util/ListUtil.java index d09cc8731..72fa268a7 100644 --- a/src/main/java/ortus/boxlang/runtime/types/util/ListUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/util/ListUtil.java @@ -80,6 +80,7 @@ public static String asString( Array list, String delimiter ) { return list.stream() // map nulls to empty string since the string caster won't do this .map( s -> s == null ? "" : s ) + .map( v -> v instanceof Array arrayValue ? "'" + arrayValue.toString() + "'" : v ) .map( StringCaster::cast ) .collect( Collectors.joining( list.containsDelimiters ? "" : delimiter ) ); } From 8ae849c48d5f133ee26191c4da124b659af16d27 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sun, 22 Dec 2024 07:37:09 -0500 Subject: [PATCH 050/161] JDBC - More list query param test work --- .../bifs/global/jdbc/QueryExecuteTest.java | 21 +++++++++++++++++-- .../runtime/components/jdbc/QueryTest.java | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index fb9290ce5..97b51eee5 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -156,9 +156,9 @@ public void testArrayBindings() { } @Disabled( "Unimplemented" ) - @DisplayName( "It can execute a query with a list binding" ) + @DisplayName( "It can execute a query with a (string) list binding" ) @Test - public void testListBindings() { + public void testListStringBindings() { instance.executeSource( """ result = queryExecute( @@ -172,6 +172,23 @@ public void testListBindings() { assertEquals( 3, query.size() ); } + @Disabled( "Unimplemented" ) + @DisplayName( "It can execute a query with an (array) list binding" ) + @Test + public void testListArrayBindings() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id IN (:ids)", + { "ids" : { value: [77, 1, 42], list : true } } + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 3, query.size() ); + } + @DisplayName( "It can execute a query with struct bindings on the default datasource" ) @Test public void testStructBindings() { diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index eb6eff6e3..10961cd00 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -165,7 +165,7 @@ public void testListBindings() { getInstance().executeSource( """ - SELECT * FROM developers WHERE id IN () + SELECT * FROM developers WHERE id IN """, context ); From 0a417ba3d070044bce6801716860a81fbe9b6e4d Mon Sep 17 00:00:00 2001 From: Michael Born Date: Mon, 23 Dec 2024 07:54:47 -0500 Subject: [PATCH 051/161] JDBC - Implement list param support Parentheses are still required around list params. --- .../boxlang/runtime/jdbc/PendingQuery.java | 60 +++++++++++-- .../boxlang/runtime/jdbc/QueryParameter.java | 89 +++++++++++++------ .../runtime/types/QueryColumnType.java | 30 +++++++ .../bifs/global/jdbc/QueryExecuteTest.java | 2 - .../runtime/components/jdbc/QueryTest.java | 4 +- 5 files changed, 144 insertions(+), 41 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index c942c1cf4..54fb50f15 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -42,6 +42,7 @@ import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.Struct; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.types.exceptions.DatabaseException; @@ -68,6 +69,8 @@ public class PendingQuery { /** * A pattern to match named parameters in the SQL string. + * + * @TODO: Move to RegexBuilder constant class. */ private static final Pattern pattern = Pattern.compile( ":\\w+" ); @@ -152,6 +155,7 @@ public PendingQuery( @Nonnull String sql, Object bindings, QueryOptions queryOpt this.sql = eventArgs.getAsString( Key.sql ); this.originalSql = eventArgs.getAsString( Key.sql ); this.parameters = processBindings( eventArgs.get( Key.of( "bindings" ) ) ); + this.sql = massageSQL(); this.queryOptions = eventArgs.getAs( QueryOptions.class, Key.options ); // Create a cache key with a default or via the passed options. @@ -259,12 +263,36 @@ private List buildParameterList( @Nonnull IStruct parameters ) { if ( paramValue == null ) { throw new DatabaseException( "Missing param in query: [" + paramName + "]. SQL: " + sql ); } - params.add( QueryParameter.fromAny( paramValue ) ); + params.add( QueryParameter.fromAny( paramName, paramValue ) ); } - this.sql = matcher.replaceAll( "?" ); return params; } + /** + * Massage the SQL string in case of list parameters. + */ + private String massageSQL() { + if ( parameters.isEmpty() ) { + return sql; + } + for ( QueryParameter p : parameters ) { + if ( p.isListParam() ) { + Array v = ( Array ) p.getValue(); + String sqlReplacement = v.stream().map( param -> "?" ).collect( Collectors.joining( "," ) ); + String placeholder = ":" + p.getName(); + + // Matcher parenPattern = Pattern.compile( "\\(\\s+" + placeholder + "\\s+\\)" ).matcher( sql ); + // if ( !parenPattern.find() ) { + // sql = sql.replaceFirst( placeholder, "(" + sqlReplacement + ")" ); + // } else { + sql = sql.replaceFirst( placeholder, sqlReplacement ); + // } + } + } + this.sql = pattern.matcher( sql ).replaceAll( "?" ); + return sql; + } + /** * Returns the original sql for this PendingQuery * @@ -434,14 +462,30 @@ private void applyParameters( Statement statement, IBoxContext context ) throws if ( statement instanceof PreparedStatement preparedStatement ) { // The param index starts from 1 - for ( int i = 1; i <= this.parameters.size(); i++ ) { - QueryParameter param = this.parameters.get( i - 1 ); - Integer scaleOrLength = param.getScaleOrLength(); - if ( scaleOrLength == null ) { - preparedStatement.setObject( i, param.toSQLType( context ), param.getSqlTypeAsInt() ); + int parameterIndex = 1; + for ( QueryParameter param : this.parameters ) { + // QueryParameter param = this.parameters.get( i - 1 ); + Integer scaleOrLength = param.getScaleOrLength(); + if ( param.isListParam() ) { + Array list = ( Array ) param.getValue(); + for ( Object value : list ) { + Object casted = QueryColumnType.toSQLType( param.getType(), value, context ); + if ( scaleOrLength == null ) { + preparedStatement.setObject( parameterIndex, casted, param.getSqlTypeAsInt() ); + } else { + preparedStatement.setObject( parameterIndex, casted, param.getSqlTypeAsInt(), + scaleOrLength ); + } + parameterIndex++; + } } else { - preparedStatement.setObject( i, param.toSQLType( context ), param.getSqlTypeAsInt(), scaleOrLength ); + if ( scaleOrLength == null ) { + preparedStatement.setObject( parameterIndex, param.toSQLType( context ), param.getSqlTypeAsInt() ); + } else { + preparedStatement.setObject( parameterIndex, param.toSQLType( context ), param.getSqlTypeAsInt(), scaleOrLength ); + } } + parameterIndex++; } } } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index f8f605190..d3f1b2ceb 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -15,13 +15,8 @@ package ortus.boxlang.runtime.jdbc; import ortus.boxlang.runtime.context.IBoxContext; -import ortus.boxlang.runtime.dynamic.casters.BigIntegerCaster; import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; -import ortus.boxlang.runtime.dynamic.casters.DateTimeCaster; -import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; -import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; -import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.dynamic.casters.StructCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Array; @@ -36,6 +31,11 @@ */ public class QueryParameter { + /** + * The parameter name. + */ + private final String name; + /** * The parameter value. */ @@ -46,6 +46,11 @@ public class QueryParameter { */ private final QueryColumnType type; + /** + * Unmodified, uncasted SQL type of the parameter as a string. (With the cf_sql_ prefix removed.) + */ + private final String sqltype; + /** * The maximum length of the parameter. Defaults to `null`. */ @@ -79,7 +84,7 @@ public class QueryParameter { *
  • `maxLength` - The maximum length of the parameter. Defaults to `null`.
    • *
  • `scale` - The scale of the parameter, used only on `double` and `decimal` types. Defaults to `null`.
    • */ - private QueryParameter( IStruct param ) { + private QueryParameter( String name, IStruct param ) { String sqltype = ( String ) param.getOrDefault( Key.sqltype, param.getOrDefault( Key.type, "VARCHAR" ) ); // allow nulls and null this.isNullParam = BooleanCaster.cast( param.getOrDefault( Key.nulls, param.getOrDefault( Key.nulls2, false ) ) ); @@ -87,7 +92,6 @@ private QueryParameter( IStruct param ) { Object v = param.get( Key.value ); if ( this.isListParam ) { - sqltype = "ARRAY"; if ( v instanceof Array ) { // do nothing? } else { @@ -95,10 +99,10 @@ private QueryParameter( IStruct param ) { } } + this.name = name; this.value = this.isNullParam ? null : v; - this.type = QueryColumnType.fromString( - RegexBuilder.of( sqltype, RegexBuilder.CF_SQL ).replaceAllAndGet( "" ) - ); + this.sqltype = RegexBuilder.of( sqltype, RegexBuilder.CF_SQL ).replaceAllAndGet( "" ).toUpperCase().trim(); + this.type = QueryColumnType.fromString( this.sqltype ); this.maxLength = param.getAsInteger( Key.maxLength ); this.scale = param.getAsInteger( Key.scale ); } @@ -110,18 +114,53 @@ private QueryParameter( IStruct param ) { * null, list, or maxLength/scale properties. */ public static QueryParameter fromAny( Object value ) { + return QueryParameter.fromAny( null, value ); + } + + /** + * Construct a new QueryParameter from a given name and value. + *

    + * If the value is an IStruct, it will be used as the construction arguments to {@link QueryParameter#QueryParameter(IStruct)}. Otherwise, the QueryParameter will be constructed with the value as the `value` + * property of the IStruct, and no sqltype, + * null, list, or maxLength/scale properties. + * + * @param name The parameter name. Null is completely valid. + * @param value The parameter value. + */ + public static QueryParameter fromAny( String name, Object value ) { CastAttempt castAsStruct = StructCaster.attempt( value ); if ( castAsStruct.wasSuccessful() ) { - return new QueryParameter( castAsStruct.getOrFail() ); + return new QueryParameter( name, castAsStruct.getOrFail() ); } - return new QueryParameter( Struct.of( "value", value ) ); + return new QueryParameter( name, Struct.of( "value", value ) ); + } + + /** + * Retrieve the parameter name. + */ + public String getName() { + return this.name; + } + + /** + * Is this a list parameter? + */ + public boolean isListParam() { + return this.isListParam; + } + + /** + * Is this parameter specifically typed as 'null'? + */ + public boolean isNullParam() { + return this.isNullParam; } /** * Retrieve the parameter value. */ public Object getValue() { - return value; + return this.value; } /** @@ -131,22 +170,14 @@ public Object toSQLType( IBoxContext context ) { if ( this.value == null ) { return null; } - return switch ( this.type ) { - case QueryColumnType.INTEGER -> IntegerCaster.cast( this.value ); - case QueryColumnType.BIGINT -> BigIntegerCaster.cast( this.value ); - case QueryColumnType.DOUBLE -> DoubleCaster.cast( this.value ); - case QueryColumnType.DECIMAL -> DoubleCaster.cast( this.value ); - case QueryColumnType.CHAR, VARCHAR -> StringCaster.cast( this.value ); - case QueryColumnType.BINARY -> this.value; // @TODO: Will this work? - case QueryColumnType.BIT -> BooleanCaster.cast( this.value ); - case QueryColumnType.BOOLEAN -> BooleanCaster.cast( this.value ); - case QueryColumnType.TIME -> DateTimeCaster.cast( this.value, context ); - case QueryColumnType.DATE -> DateTimeCaster.cast( this.value, context ); - case QueryColumnType.TIMESTAMP -> new java.sql.Timestamp( DateTimeCaster.cast( this.value, context ).toEpochMillis() ); - case QueryColumnType.OBJECT -> this.value; - case QueryColumnType.OTHER -> this.value; - case QueryColumnType.NULL -> null; - }; + return QueryColumnType.toSQLType( this.type, this.value, context ); + } + + /** + * Retrieve the QueryColumnType of the parameter. + */ + public QueryColumnType getType() { + return this.type; } /** diff --git a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java index 77d3dc708..ef7a3aa62 100644 --- a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java +++ b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java @@ -19,6 +19,14 @@ import java.sql.Types; +import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.BigIntegerCaster; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; +import ortus.boxlang.runtime.dynamic.casters.DateTimeCaster; +import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; + /** * Represents a column type in a Query object. */ @@ -251,6 +259,28 @@ public static QueryColumnType fromSQLType( int type ) { } } + public static Object toSQLType( QueryColumnType type, Object value, IBoxContext context ) { + if ( value == null ) { + return null; + } + return switch ( type ) { + case QueryColumnType.INTEGER -> IntegerCaster.cast( value ); + case QueryColumnType.BIGINT -> BigIntegerCaster.cast( value ); + case QueryColumnType.DOUBLE -> DoubleCaster.cast( value ); + case QueryColumnType.DECIMAL -> DoubleCaster.cast( value ); + case QueryColumnType.CHAR, VARCHAR -> StringCaster.cast( value ); + case QueryColumnType.BINARY -> value; // @TODO: Will this work? + case QueryColumnType.BIT -> BooleanCaster.cast( value ); + case QueryColumnType.BOOLEAN -> BooleanCaster.cast( value ); + case QueryColumnType.TIME -> DateTimeCaster.cast( value, context ); + case QueryColumnType.DATE -> DateTimeCaster.cast( value, context ); + case QueryColumnType.TIMESTAMP -> new java.sql.Timestamp( DateTimeCaster.cast( value, context ).toEpochMillis() ); + case QueryColumnType.OBJECT -> value; + case QueryColumnType.OTHER -> value; + case QueryColumnType.NULL -> null; + }; + } + /** * Does this represent a type that can be treated as a string? * diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index 97b51eee5..ac9208106 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -155,7 +155,6 @@ public void testArrayBindings() { assertEquals( "Developer", michael.get( "role" ) ); } - @Disabled( "Unimplemented" ) @DisplayName( "It can execute a query with a (string) list binding" ) @Test public void testListStringBindings() { @@ -172,7 +171,6 @@ public void testListStringBindings() { assertEquals( 3, query.size() ); } - @Disabled( "Unimplemented" ) @DisplayName( "It can execute a query with an (array) list binding" ) @Test public void testListArrayBindings() { diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index 10961cd00..2af683cb2 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -158,14 +158,14 @@ public void testArrayBindings() { assertEquals( "Developer", michael.get( "role" ) ); } - @Disabled( "Unimplemented" ) + @Disabled( "Parsing error!" ) @DisplayName( "It can execute a query with a list queryparam" ) @Test public void testListBindings() { getInstance().executeSource( """ - SELECT * FROM developers WHERE id IN + SELECT * FROM developers WHERE id IN """, context ); From 79701a91f491d2f4e22977c4c951c20c5e363426 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Mon, 23 Dec 2024 08:12:17 -0500 Subject: [PATCH 052/161] JDBC - Javadocs and SQL regex cleanup --- .../ortus/boxlang/runtime/jdbc/DataSource.java | 6 ++++++ .../ortus/boxlang/runtime/jdbc/PendingQuery.java | 14 +++----------- .../boxlang/runtime/types/QueryColumnType.java | 10 ++++++++++ .../ortus/boxlang/runtime/util/RegexBuilder.java | 1 + 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java b/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java index 740c4c6ee..d8bacd7db 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java @@ -251,6 +251,12 @@ public ExecutedQuery execute( String query ) { } } + /** + * Execute a query on the default connection, within the specific context. + * + * @param query The SQL query to execute. + * @param context The boxlang context for localization. Useful for localization; i.e., queries with date or time values. + */ public ExecutedQuery execute( String query, IBoxContext context ) { try ( Connection conn = getConnection() ) { return execute( query, conn, context ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index 54fb50f15..1b1c831b6 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -47,6 +46,7 @@ import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.types.exceptions.DatabaseException; import ortus.boxlang.runtime.types.util.ListUtil; +import ortus.boxlang.runtime.util.RegexBuilder; /** * This class represents a query and any parameters/bindings before being executed. @@ -67,13 +67,6 @@ public class PendingQuery { */ private static final InterceptorService interceptorService = BoxRuntime.getInstance().getInterceptorService(); - /** - * A pattern to match named parameters in the SQL string. - * - * @TODO: Move to RegexBuilder constant class. - */ - private static final Pattern pattern = Pattern.compile( ":\\w+" ); - /** * Prefix for cache queries */ @@ -255,7 +248,7 @@ private List buildParameterList( @Nonnull Array parameters ) { */ private List buildParameterList( @Nonnull IStruct parameters ) { List params = new ArrayList<>(); - Matcher matcher = pattern.matcher( sql ); + Matcher matcher = RegexBuilder.SQL_PARAMETER.matcher( sql ); while ( matcher.find() ) { String paramName = matcher.group(); paramName = paramName.substring( 1 ); @@ -289,7 +282,7 @@ private String massageSQL() { // } } } - this.sql = pattern.matcher( sql ).replaceAll( "?" ); + this.sql = RegexBuilder.of( sql, RegexBuilder.SQL_PARAMETER ).replaceAllAndGet( "?" ); return sql; } @@ -464,7 +457,6 @@ private void applyParameters( Statement statement, IBoxContext context ) throws // The param index starts from 1 int parameterIndex = 1; for ( QueryParameter param : this.parameters ) { - // QueryParameter param = this.parameters.get( i - 1 ); Integer scaleOrLength = param.getScaleOrLength(); if ( param.isListParam() ) { Array list = ( Array ) param.getValue(); diff --git a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java index ef7a3aa62..cd45b61df 100644 --- a/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java +++ b/src/main/java/ortus/boxlang/runtime/types/QueryColumnType.java @@ -259,6 +259,16 @@ public static QueryColumnType fromSQLType( int type ) { } } + /** + * Convert a value to the appropriate SQL type. + *

    + * + * @TODO: This may better belong in a Caster class. + * + * @param type The query column type to convert to. + * @param value The value to convert. + * @param context The context in which the conversion is taking place. Useful for localization. + */ public static Object toSQLType( QueryColumnType type, Object value, IBoxContext context ) { if ( value == null ) { return null; diff --git a/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java b/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java index ba5e57b7e..2d0729763 100644 --- a/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java +++ b/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java @@ -78,6 +78,7 @@ public class RegexBuilder { public static final Pattern VALID_VARIABLENAME = Pattern.compile( "^[a-zA-Z_][a-zA-Z0-9_]*$" ); public static final Pattern WHITESPACE = Pattern.compile( "\\s" ); public static final Pattern ZIPCODE = Pattern.compile( "\\d{5}([ -]?\\d{4})?" ); + public static final Pattern SQL_PARAMETER = Pattern.compile( ":\\w+" ); /** * Build a matcher for the given pattern lookup From 97568c8c0fb5076b2d6fdc105766454f3fcd4b8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:37:49 +0000 Subject: [PATCH 053/161] Bump com.github.javaparser:javaparser-symbol-solver-core Bumps [com.github.javaparser:javaparser-symbol-solver-core](https://github.com/javaparser/javaparser) from 3.26.2 to 3.26.3. - [Release notes](https://github.com/javaparser/javaparser/releases) - [Changelog](https://github.com/javaparser/javaparser/blob/master/changelog.md) - [Commits](https://github.com/javaparser/javaparser/compare/javaparser-parent-3.26.2...javaparser-parent-3.26.3) --- updated-dependencies: - dependency-name: com.github.javaparser:javaparser-symbol-solver-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4f9ec979c..551fbce70 100644 --- a/build.gradle +++ b/build.gradle @@ -109,7 +109,7 @@ dependencies { // https://mvnrepository.com/artifact/commons-io/commons-io implementation "commons-io:commons-io:2.18.0" // https://mvnrepository.com/artifact/com.github.javaparser/javaparser-symbol-solver-core - implementation 'com.github.javaparser:javaparser-symbol-solver-core:3.26.2' + implementation 'com.github.javaparser:javaparser-symbol-solver-core:3.26.3' // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 implementation 'org.apache.commons:commons-lang3:3.17.0' // https://mvnrepository.com/artifact/org.apache.commons/commons-text From ef075ec3bfe11b9836a1456c045a3c35577160c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:38:01 +0000 Subject: [PATCH 054/161] Bump ch.qos.logback:logback-classic from 1.5.12 to 1.5.15 Bumps [ch.qos.logback:logback-classic](https://github.com/qos-ch/logback) from 1.5.12 to 1.5.15. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.12...v_1.5.15) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-classic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4f9ec979c..4565f5b20 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ dependencies { // https://mvnrepository.com/artifact/org.slf4j/slf4j-api implementation 'org.slf4j:slf4j-api:2.0.16' // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic - implementation 'ch.qos.logback:logback-classic:1.5.12' + implementation 'ch.qos.logback:logback-classic:1.5.15' // https://mvnrepository.com/artifact/com.zaxxer/HikariCP implementation 'com.zaxxer:HikariCP:6.2.1' // https://mvnrepository.com/artifact/org.ow2.asm/asm-tree From b8ebd59513567c829ee10bc7bad36954ba067dce Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 23 Dec 2024 11:32:31 -0600 Subject: [PATCH 055/161] BL-880 --- .../ortus/boxlang/runtime/types/Query.java | 21 +++++++++++++++++- .../bifs/global/query/QueryNewTest.java | 22 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index ec0ea82e1..97da56525 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -419,6 +419,25 @@ public int addRow( Object[] row ) { return newRow; } + /** + * Add a row to the query. If the array has fewer items than columns in the query, add nulls for the missing values. + * + * @param row row data as array of objects + * + * @return this query + */ + public int addRowDefaultMissing( Object[] row ) { + if ( row.length < columns.size() ) { + Object[] newRow = new Object[ columns.size() ]; + System.arraycopy( row, 0, newRow, 0, row.length ); + for ( int i = row.length; i < columns.size(); i++ ) { + newRow[ i ] = null; + } + row = newRow; + } + return addRow( row ); + } + /** * Add a row to the query * @@ -427,7 +446,7 @@ public int addRow( Object[] row ) { * @return this query */ public int addRow( Array row ) { - return addRow( row.toArray() ); + return addRowDefaultMissing( row.toArray() ); } /** diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java index 800d2b5da..234ae8a7d 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java @@ -222,4 +222,26 @@ public void testTrimColumnNames() { assertThat( variables.get( result ) ).isInstanceOf( Query.class ); assertThat( variables.get( "columnList" ) ).isEqualTo( "id,name" ); } + + @DisplayName( "It defaults missing data" ) + @Test + public void testDefaultsMissingData() { + instance.executeSource( + """ + import java.util.Arrays; + + result = QueryNew( "column1", "VarChar", [ [], [ "a" ] ] ) + println( Arrays.toString( result.getData().get(0) ) ) + println( Arrays.toString( result.getData().get(1) ) ) + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query qry = variables.getAsQuery( result ); + assertThat( qry.size() ).isEqualTo( 2 ); + assertThat( qry.getRow( 0 ).length ).isEqualTo( 1 ); + assertThat( qry.getRow( 0 )[ 0 ] ).isNull(); + assertThat( qry.getRow( 1 ).length ).isEqualTo( 1 ); + assertThat( qry.getRow( 1 )[ 0 ] ).isEqualTo( "a" ); + } + } From 883e431a449c275788567af155a7d03bdd9dbeaa Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 23 Dec 2024 11:52:33 -0600 Subject: [PATCH 056/161] BL-879 --- .../runtime/dynamic/casters/NumberCaster.java | 26 ++++++++++------- .../bifs/global/query/QueryNewTest.java | 2 -- .../dynamic/casters/NumberCasterTest.java | 28 ++++++------------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java index b233fae61..0ccdf8278 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java @@ -32,6 +32,8 @@ */ public class NumberCaster implements IBoxCaster { + public static boolean booleansAreNumbers = false; + /** * Tests to see if the value can be cast to a Number. * Returns a {@code CastAttempt} which will contain the result if casting was @@ -91,19 +93,23 @@ public static Number cast( Object object, Boolean fail ) { return new BigDecimal( num.doubleValue(), MathUtil.getMathContext() ); } - if ( object instanceof Boolean bool ) { - return bool ? 1 : 0; - } + // Only here for compat. This "hidden setting" can be toggled by the compat module + if ( booleansAreNumbers ) { + if ( object instanceof Boolean bool ) { + return bool ? 1 : 0; + } - if ( object instanceof String str ) { - // String true and yes are truthy - if ( str.equalsIgnoreCase( "true" ) || str.equalsIgnoreCase( "yes" ) ) { - return 1; - // String false and no are truthy - } else if ( str.equalsIgnoreCase( "false" ) || str.equalsIgnoreCase( "no" ) ) { - return 0; + if ( object instanceof String str ) { + // String true and yes are truthy + if ( str.equalsIgnoreCase( "true" ) || str.equalsIgnoreCase( "yes" ) ) { + return 1; + // String false and no are truthy + } else if ( str.equalsIgnoreCase( "false" ) || str.equalsIgnoreCase( "no" ) ) { + return 0; + } } } + // Try to parse the string as a Number String stringValue = StringCaster.cast( object, false ); Number result = parseNumber( stringValue ); diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java index 234ae8a7d..2704b9712 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/query/QueryNewTest.java @@ -231,8 +231,6 @@ public void testDefaultsMissingData() { import java.util.Arrays; result = QueryNew( "column1", "VarChar", [ [], [ "a" ] ] ) - println( Arrays.toString( result.getData().get(0) ) ) - println( Arrays.toString( result.getData().get(1) ) ) """, context ); assertThat( variables.get( result ) ).isInstanceOf( Query.class ); diff --git a/src/test/java/ortus/boxlang/runtime/dynamic/casters/NumberCasterTest.java b/src/test/java/ortus/boxlang/runtime/dynamic/casters/NumberCasterTest.java index 8fd5e1b9f..6b3003f65 100644 --- a/src/test/java/ortus/boxlang/runtime/dynamic/casters/NumberCasterTest.java +++ b/src/test/java/ortus/boxlang/runtime/dynamic/casters/NumberCasterTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import ortus.boxlang.runtime.types.exceptions.BoxCastException; import ortus.boxlang.runtime.types.exceptions.BoxLangException; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -100,32 +101,21 @@ void testItCanCastACharArray() { assertThat( result.doubleValue() ).isEqualTo( 12345 ); } - @DisplayName( "It can cast a boolean to a Number" ) + @DisplayName( "It will NOT cast a boolean to a Number" ) @Test void testItCanCastABoolean() { - Number result = NumberCaster.cast( true ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 1 ); + assertThrows( BoxCastException.class, () -> NumberCaster.cast( true ) ); - result = NumberCaster.cast( false ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 0 ); + assertThrows( BoxCastException.class, () -> NumberCaster.cast( false ) ); - result = NumberCaster.cast( "true" ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 1 ); + assertThrows( BoxCastException.class, () -> NumberCaster.cast( "true" ) ); - result = NumberCaster.cast( "false" ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 0 ); + assertThrows( BoxCastException.class, () -> NumberCaster.cast( "false" ) ); - result = NumberCaster.cast( "yes" ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 1 ); + assertThrows( BoxCastException.class, () -> NumberCaster.cast( "yes" ) ); + + assertThrows( BoxCastException.class, () -> NumberCaster.cast( "no" ) ); - result = NumberCaster.cast( "no" ); - assertThat( result ).isInstanceOf( Integer.class ); - assertThat( result.doubleValue() ).isEqualTo( 0 ); } @DisplayName( "It can attempt to cast" ) From ae1fb60531dd48f55c3626af6135c4954a734704 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 23 Dec 2024 12:35:34 -0600 Subject: [PATCH 057/161] BL-879 fix tests --- .../dynamic/casters/BooleanCaster.java | 13 +++++++++++ .../boxlang/runtime/operators/Compare.java | 22 +++++++++++-------- .../ortus/boxlang/compiler/TestExecution.java | 2 +- .../ortus/boxlang/runtime/BoxRuntimeTest.java | 2 +- .../bifs/global/decision/IsNumericTest.java | 17 ++++++++++++++ .../boxlang/runtime/operators/NegateTest.java | 11 ---------- 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/BooleanCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/BooleanCaster.java index d08b96f77..26387d59b 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/BooleanCaster.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/BooleanCaster.java @@ -60,6 +60,19 @@ public static CastAttempt attempt( Object object ) { return CastAttempt.ofNullable( cast( object, false ) ); } + /** + * Tests to see if the value can be cast to a boolean. + * Returns a {@code CastAttempt} which will contain the result if casting was successful, + * or can be interrogated to proceed otherwise. + * + * @param object The value to cast to a boolean + * + * @return The boolean value + */ + public static CastAttempt attempt( Object object, Boolean loose ) { + return CastAttempt.ofNullable( cast( object, false, loose ) ); + } + /** * Used to cast anything to a boolean, throwing exception if we fail * diff --git a/src/main/java/ortus/boxlang/runtime/operators/Compare.java b/src/main/java/ortus/boxlang/runtime/operators/Compare.java index 9679d1031..140279292 100644 --- a/src/main/java/ortus/boxlang/runtime/operators/Compare.java +++ b/src/main/java/ortus/boxlang/runtime/operators/Compare.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import ortus.boxlang.runtime.dynamic.casters.BigDecimalCaster; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; import ortus.boxlang.runtime.dynamic.casters.DateTimeCaster; import ortus.boxlang.runtime.dynamic.casters.NumberCaster; @@ -132,19 +133,21 @@ public static Integer attempt( Object left, Object right, Boolean caseSensitive, } } + // Check boolean + CastAttempt leftBooleanAttempt = BooleanCaster.attempt( left, false ); + if ( leftBooleanAttempt.wasSuccessful() ) { + CastAttempt rightBooleanAttempt = BooleanCaster.attempt( right, false ); + + if ( rightBooleanAttempt.wasSuccessful() ) { + return Boolean.compare( leftBooleanAttempt.get(), rightBooleanAttempt.get() ); + } + } + // String comparison if ( left instanceof String || right instanceof String ) { if ( !caseSensitive ) { left = StringUtils.lowerCase( left.toString(), locale ); right = StringUtils.lowerCase( right.toString(), locale ); - } else { - // Assume that if the case sensitive argument is passed as false, dates are not expected - if ( DateTimeCaster.attempt( left ).wasSuccessful() && DateTimeCaster.attempt( right ).wasSuccessful() ) { - // TODO: This is potentially slow with multiple failures - we need to add some validation methods in to the DateTime cast attempt - DateTime ref = DateTimeCaster.cast( left ); - DateTime target = DateTimeCaster.cast( right ); - return ref.compareTo( target ); - } } return StringCompare.invoke( StringCaster.cast( left ), StringCaster.cast( right ), caseSensitive ); @@ -152,7 +155,8 @@ public static Integer attempt( Object left, Object right, Boolean caseSensitive, } // Fallback, see if both objects are comparable - if ( left instanceof Comparable && right instanceof Comparable ) { + if ( left instanceof Comparable && right instanceof Comparable + && ( left.getClass().isAssignableFrom( right.getClass() ) || right.getClass().isAssignableFrom( left.getClass() ) ) ) { return caseSensitive && left instanceof Key keyLeft && right instanceof Key keyRight diff --git a/src/test/java/ortus/boxlang/compiler/TestExecution.java b/src/test/java/ortus/boxlang/compiler/TestExecution.java index 7940d9ab0..e98864586 100644 --- a/src/test/java/ortus/boxlang/compiler/TestExecution.java +++ b/src/test/java/ortus/boxlang/compiler/TestExecution.java @@ -67,7 +67,7 @@ public void executeWhile() throws IOException { break; } } - if(!a % 2 == 0) { + if(!(a % 2 == 0)) { } a +=1; diff --git a/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java b/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java index 934fcb3c2..af2edb43b 100644 --- a/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java +++ b/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java @@ -156,7 +156,7 @@ public void testItCanExecuteMoreStatements() { break; } } - if(!a % 2 == 0) { + if(!(a % 2 == 0)) { } a +=1; diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java index 0e8309cb1..e8509933a 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java @@ -93,6 +93,23 @@ public void testSimpleNumerics() { assertThat( ( Boolean ) variables.get( Key.of( "struct" ) ) ).isFalse(); } + @DisplayName( "It returns false for booleans" ) + @Test + public void testBooleans() { + instance.executeSource( + """ + booltrue = IsNumeric( true ) + boolfalse = IsNumeric( false ) + stringtrue = IsNumeric( "true" ) + stringfalse = IsNumeric( "false" ) + """, + context ); + assertThat( variables.getAsBoolean( Key.of( "booltrue" ) ) ).isFalse(); + assertThat( variables.getAsBoolean( Key.of( "boolfalse" ) ) ).isFalse(); + assertThat( variables.getAsBoolean( Key.of( "stringtrue" ) ) ).isFalse(); + assertThat( variables.getAsBoolean( Key.of( "stringfalse" ) ) ).isFalse(); + } + @DisplayName( "It tests the BIF IsNumeric with locale arguments" ) @Test public void testWithLocale() { diff --git a/src/test/java/ortus/boxlang/runtime/operators/NegateTest.java b/src/test/java/ortus/boxlang/runtime/operators/NegateTest.java index 2783145c3..71dbb34e8 100644 --- a/src/test/java/ortus/boxlang/runtime/operators/NegateTest.java +++ b/src/test/java/ortus/boxlang/runtime/operators/NegateTest.java @@ -38,15 +38,4 @@ void testItCanNegateString() { assertThat( Negate.invoke( "-5" ) ).isEqualTo( 5 ); } - @DisplayName( "It can mathematically negate a boolean" ) - @Test - void testItCanNegateBoolean() { - assertThat( Negate.invoke( true ) ).isEqualTo( -1 ); - assertThat( Negate.invoke( false ) ).isEqualTo( 0 ); - assertThat( Negate.invoke( "true" ) ).isEqualTo( -1 ); - assertThat( Negate.invoke( "false" ) ).isEqualTo( 0 ); - assertThat( Negate.invoke( "yes" ) ).isEqualTo( -1 ); - assertThat( Negate.invoke( "no" ) ).isEqualTo( 0 ); - } - } From f05e777fec811713c166c52df187209a8c56a30a Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 23 Dec 2024 14:22:46 -0600 Subject: [PATCH 058/161] BL-823 perf fixes first round --- .../ast/sql/select/expression/SQLColumn.java | 68 +++++++++++++++++-- .../runtime/jdbc/qoq/QoQFunctionService.java | 5 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 5 +- .../jdbc/qoq/functions/scalar/LCase.java | 31 +++++++++ .../jdbc/qoq/functions/scalar/UCase.java | 31 +++++++++ .../ortus/boxlang/runtime/types/Query.java | 18 +++-- 6 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/LCase.java create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/UCase.java diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java index 5ae99ed76..e0f535114 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java @@ -25,6 +25,8 @@ import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.QueryColumn; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -40,6 +42,12 @@ public class SQLColumn extends SQLExpression { private Key name; + // THIS DATA IS SPECIFIC TO THE EXECUTION. IF WE START CACHING THIS AST, REWORK THIS TO BE STORED ELSEWHERE + private List data = null; + private int tableIndex = -1; + private int colIndex = -1; + private QueryColumnType type = null; + /** * Constructor * @@ -95,6 +103,41 @@ public SQLTable getTableFinal( QoQSelectExecution QoQExec ) { throw new BoxRuntimeException( "Column " + name + " is ambiguous and not found in any table." ); } + /** + * Get the table, performing runtime lookup if necessary + */ + public void ensureData( QoQSelectExecution QoQExec ) { + if ( data != null ) { + return; + } + + synchronized ( this ) { + var t = getTable(); + if ( data != null ) { + return; + } + if ( t == null ) { + // Abmiguity, we need to find the table + var tables = QoQExec.getTableLookup().entrySet(); + for ( var tableSet : tables ) { + if ( tableSet.getValue().getColumns().containsKey( name ) ) { + t = tableSet.getKey(); + } + } + if ( t == null ) { + throw new BoxRuntimeException( "Column " + name + " is ambiguous and not found in any table." ); + } + } + tableIndex = t.getIndex(); + Query table = QoQExec.getTableLookup().get( t ); + // Cache this data for future use + QueryColumn column = table.getColumns().get( name ); + colIndex = column.getIndex(); + type = column.getType(); + data = table.getData(); + } + } + /** * Set the table */ @@ -107,22 +150,35 @@ public void setTable( SQLTable table ) { * What type does this expression evaluate to */ public QueryColumnType getType( QoQSelectExecution QoQExec ) { - return QoQExec.getTableLookup().get( getTableFinal( QoQExec ) ).getColumns().get( name ).getType(); + ensureData( QoQExec ); + return type; + // return QoQExec.getTableLookup().get( getTableFinal( QoQExec ) ).getColumns().get( name ).getType(); } /** * Evaluate the expression */ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { - var tableFinal = getTableFinal( QoQExec ); - // System.out.println( "getting SQL column: " + name.getName() + " from table: " + tableFinal.getName() + " with index: " + tableFinal.getIndex() ); - // System.out.println( "intersection: " + Arrays.toString( intersection ) ); - int rowNum = intersection[ tableFinal.getIndex() ]; + ensureData( QoQExec ); + int rowNum = intersection[ tableIndex ]; + // This means an outer join matched nothing if ( rowNum == 0 ) { return null; } - return QoQExec.getTableLookup().get( tableFinal ).getCell( name, rowNum - 1 ); + + return data.get( intersection[ tableIndex ] - 1 )[ colIndex ]; + /* + * var tableFinal = getTableFinal( QoQExec ); + * // System.out.println( "getting SQL column: " + name.getName() + " from table: " + tableFinal.getName() + " with index: " + tableFinal.getIndex() ); + * // System.out.println( "intersection: " + Arrays.toString( intersection ) ); + * int rowNum = intersection[ tableFinal.getIndex() ]; + * // This means an outer join matched nothing + * if ( rowNum == 0 ) { + * return null; + * } + * return QoQExec.getTableLookup().get( tableFinal ).getCell( name, rowNum - 1 ); + */ } /** diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java index d7fba8ac1..2ff2ad29b 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java @@ -39,6 +39,7 @@ import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Exp; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Floor; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.IsNull; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.LCase; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Left; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Length; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Lower; @@ -51,6 +52,7 @@ import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Sqrt; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Tan; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Trim; +import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.UCase; import ortus.boxlang.runtime.jdbc.qoq.functions.scalar.Upper; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.QueryColumnType; @@ -67,7 +69,9 @@ public class QoQFunctionService { // Scalar register( Upper.INSTANCE ); + register( UCase.INSTANCE ); register( Lower.INSTANCE ); + register( LCase.INSTANCE ); register( Abs.INSTANCE ); register( Acos.INSTANCE ); register( Asin.INSTANCE ); @@ -80,7 +84,6 @@ public class QoQFunctionService { register( Floor.INSTANCE ); register( IsNull.INSTANCE ); register( Length.INSTANCE ); - register( Lower.INSTANCE ); register( Ltrim.INSTANCE ); register( Mod.INSTANCE ); register( Power.INSTANCE ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index d92818f34..a27522cb2 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -270,7 +270,10 @@ public Query getIndepententSubQuery( SQLSelectStatement subquery ) { * @param partition The partition */ public void addPartition( String partitionName, int[] partition ) { - partitions.computeIfAbsent( partitionName, p -> new ArrayList() ).add( partition ); + List thisPartition = partitions.computeIfAbsent( partitionName, p -> new ArrayList() ); + synchronized ( thisPartition ) { + thisPartition.add( partition ); + } } /** diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/LCase.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/LCase.java new file mode 100644 index 000000000..796929922 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/LCase.java @@ -0,0 +1,31 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.scopes.Key; + +public class LCase extends Lower { + + private static final Key name = Key.of( "lcase" ); + + public static final QoQScalarFunctionDef INSTANCE = new LCase(); + + @Override + public Key getName() { + return name; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/UCase.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/UCase.java new file mode 100644 index 000000000..0b15584a3 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/UCase.java @@ -0,0 +1,31 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq.functions.scalar; + +import ortus.boxlang.runtime.jdbc.qoq.QoQScalarFunctionDef; +import ortus.boxlang.runtime.scopes.Key; + +public class UCase extends Upper { + + private static final Key name = Key.of( "ucase" ); + + public static final QoQScalarFunctionDef INSTANCE = new UCase(); + + @Override + public Key getName() { + return name; + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index 97da56525..24bd20cab 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -62,7 +62,8 @@ public class Query implements IType, IReferenceable, Collection, Serial /** * Query data as List of arrays */ - private List data = Collections.synchronizedList( new ArrayList() ); + // private List data = Collections.synchronizedList( new ArrayList() ); + private List data = new ArrayList(); /** * Map of column definitions @@ -787,7 +788,9 @@ public boolean add( IStruct row ) { @Override public boolean remove( Object o ) { - return data.remove( o ); + synchronized ( data ) { + return data.remove( o ); + } } @Override @@ -805,12 +808,16 @@ public boolean addAll( Collection rows ) { @Override public boolean removeAll( Collection c ) { - return data.removeAll( c ); + synchronized ( data ) { + return data.removeAll( c ); + } } @Override public boolean retainAll( Collection c ) { - return data.retainAll( c ); + synchronized ( data ) { + return data.retainAll( c ); + } } @Override @@ -972,7 +979,8 @@ public Query duplicate( boolean deep ) { @Override public int hashCode() { - return computeHashCode( IType.createIdentitySetForType() ); + return super.hashCode(); + // return computeHashCode( IType.createIdentitySetForType() ); } @Override From 2c8f414d90b7d21c72078fa1ec777005e47d6ce0 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 26 Dec 2024 17:32:59 -0600 Subject: [PATCH 059/161] BL-823 --- .../ast/sql/select/expression/SQLCase.java | 10 +- .../select/expression/SQLCaseWhenThen.java | 3 +- .../ast/sql/select/expression/SQLColumn.java | 8 +- .../select/expression/SQLCountFunction.java | 3 +- .../sql/select/expression/SQLFunction.java | 3 +- .../ast/sql/select/expression/SQLOrderBy.java | 3 +- .../ast/sql/select/expression/SQLParam.java | 3 +- .../sql/select/expression/SQLParenthesis.java | 3 +- .../select/expression/SQLStarExpression.java | 3 +- .../expression/literal/SQLBooleanLiteral.java | 3 +- .../expression/literal/SQLNullLiteral.java | 3 +- .../expression/literal/SQLNumberLiteral.java | 3 +- .../expression/literal/SQLStringLiteral.java | 3 +- .../operation/SQLBetweenOperation.java | 20 +- .../operation/SQLBinaryOperation.java | 83 ++++--- .../expression/operation/SQLInOperation.java | 7 +- .../operation/SQLInSubQueryOperation.java | 7 +- .../operation/SQLUnaryOperation.java | 3 +- .../ast/visitor/PrettyPrintBoxVisitor.java | 228 ++++++++++++++++++ .../compiler/ast/visitor/VoidBoxVisitor.java | 97 ++++++++ .../compiler/toolchain/SQLVisitor.java | 2 +- .../boxlang/runtime/jdbc/qoq/QoQCompare.java | 106 ++++++++ .../runtime/jdbc/qoq/QoQExecutionService.java | 114 +++++---- .../jdbc/qoq/QoQIntersectionGenerator.java | 4 +- .../runtime/jdbc/qoq/QoQSelectExecution.java | 40 ++- .../jdbc/qoq/QoQSelectStatementExecution.java | 25 ++ .../runtime/operators/StringCompare.java | 2 +- .../ortus/boxlang/runtime/scopes/Key.java | 1 + .../ortus/boxlang/runtime/types/Query.java | 155 +++++++++--- .../types/unmodifiable/UnmodifiableQuery.java | 10 + 30 files changed, 782 insertions(+), 173 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQCompare.java diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java index 25571fce0..3ba0ed438 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCase.java @@ -21,8 +21,8 @@ import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQCompare; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; -import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -198,14 +198,13 @@ private Object processInputCase( QoQSelectExecution QoQExec, int[] intersection, } for ( SQLCaseWhenThen whenThen : whenThens ) { - Boolean result; - Object caseValue; + Object caseValue; if ( intersection != null ) { caseValue = whenThen.getWhenExpression().evaluate( QoQExec, intersection ); } else { caseValue = whenThen.getWhenExpression().evaluateAggregate( QoQExec, intersections ); } - if ( Compare.invoke( inputValue, caseValue ) == 0 ) { + if ( QoQCompare.invoke( inputExpression.getType( QoQExec ), inputValue, caseValue ) == 0 ) { return whenThen.getThenExpression().evaluate( QoQExec, intersection ); } } @@ -217,8 +216,7 @@ private Object processInputCase( QoQSelectExecution QoQExec, int[] intersection, @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java index 8f59375ef..9b9c10294 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCaseWhenThen.java @@ -76,8 +76,7 @@ public void setThenExpression( SQLExpression thenExpression ) { @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java index e0f535114..6136ab7af 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLColumn.java @@ -21,6 +21,7 @@ import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.Position; import ortus.boxlang.compiler.ast.sql.select.SQLTable; +import ortus.boxlang.compiler.ast.sql.select.SQLTableVariable; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; @@ -132,6 +133,10 @@ public void ensureData( QoQSelectExecution QoQExec ) { Query table = QoQExec.getTableLookup().get( t ); // Cache this data for future use QueryColumn column = table.getColumns().get( name ); + if ( column == null ) { + throw new BoxRuntimeException( + "Column " + name + " not found in table " + ( t instanceof SQLTableVariable tv ? tv.getName() : t.getAlias() ) ); + } colIndex = column.getIndex(); type = column.getType(); data = table.getData(); @@ -215,8 +220,7 @@ public boolean isNumeric( QoQSelectExecution QoQExec ) { @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java index 6e4e56a8e..5e68249ba 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLCountFunction.java @@ -123,8 +123,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java index 5b540aabd..ccc155195 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLFunction.java @@ -171,8 +171,7 @@ protected Object[] buildAggregateValues( QoQSelectExecution QoQExec, List @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLOrderBy.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLOrderBy.java index 1248af906..2c31b8e4b 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLOrderBy.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLOrderBy.java @@ -75,8 +75,7 @@ public void setAscending( boolean ascending ) { @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java index 6fb1794aa..a8f4d4cdd 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParam.java @@ -133,8 +133,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java index 7d7c3b9d0..8bc1ef64f 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLParenthesis.java @@ -103,8 +103,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java index a6d827d1a..c6ecca0ed 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/SQLStarExpression.java @@ -74,8 +74,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java index 9b9bb7a14..75a9875fd 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLBooleanLiteral.java @@ -89,8 +89,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java index 8dbd4c97a..9d16e1ae5 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNullLiteral.java @@ -62,8 +62,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java index d5717ff7d..e2db77f8d 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLNumberLiteral.java @@ -110,8 +110,7 @@ public boolean isNumeric( Map tableLookup ) { @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java index 16f1499a9..f10470474 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/literal/SQLStringLiteral.java @@ -86,8 +86,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java index 2bcb399d7..0df950416 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBetweenOperation.java @@ -22,8 +22,9 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQCompare; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; -import ortus.boxlang.runtime.operators.Compare; +import ortus.boxlang.runtime.types.QueryColumnType; /** * Abstract Node class representing SQL BETWEEN operation @@ -133,7 +134,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Object rightValue = right.evaluate( QoQExec, intersection ); Object expressionValue = expression.evaluate( QoQExec, intersection ); // The ^ not inverses the result if the not flag is true - return doBetween( leftValue, rightValue, expressionValue ) ^ not; + return doBetween( left.getType( QoQExec ), leftValue, rightValue, expressionValue ) ^ not; } /** @@ -143,7 +144,11 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse if ( intersections.isEmpty() ) { return false; } - return evaluate( QoQExec, intersections.get( 0 ) ); + Object leftValue = left.evaluateAggregate( QoQExec, intersections ); + Object rightValue = right.evaluateAggregate( QoQExec, intersections ); + Object expressionValue = expression.evaluateAggregate( QoQExec, intersections ); + // The ^ not inverses the result if the not flag is true + return doBetween( left.getType( QoQExec ), leftValue, rightValue, expressionValue ) ^ not; } /** @@ -155,19 +160,18 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse * * @return true if the value is between the left and right operands */ - private boolean doBetween( Object left, Object right, Object value ) { - int result = Compare.invoke( left, value, true ); + private boolean doBetween( QueryColumnType type, Object left, Object right, Object value ) { + int result = QoQCompare.invoke( type, left, value ); if ( result == 1 ) { return false; } - result = Compare.invoke( value, right, true ); + result = QoQCompare.invoke( type, value, right ); return result != 1; } @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java index e92461af0..523c559c0 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLBinaryOperation.java @@ -25,12 +25,12 @@ import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLStringLiteral; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.dynamic.casters.NumberCaster; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.jdbc.qoq.LikeOperation; +import ortus.boxlang.runtime.jdbc.qoq.QoQCompare; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; -import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.operators.Concat; -import ortus.boxlang.runtime.operators.EqualsEquals; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -202,7 +202,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { // Implement each binary operator switch ( operator ) { case DIVIDE : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -215,19 +215,27 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { case EQUAL : leftValue = left.evaluate( QoQExec, intersection ); rightValue = right.evaluate( QoQExec, intersection ); - return EqualsEquals.invoke( leftValue, rightValue, true ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) == 0; case GREATERTHAN : - return Compare.invoke( left.evaluate( QoQExec, intersection ), right.evaluate( QoQExec, intersection ), true ) == 1; + leftValue = left.evaluate( QoQExec, intersection ); + rightValue = right.evaluate( QoQExec, intersection ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) == 1; case GREATERTHANOREQUAL : - compareResult = Compare.invoke( left.evaluate( QoQExec, intersection ), right.evaluate( QoQExec, intersection ), true ); + leftValue = left.evaluate( QoQExec, intersection ); + rightValue = right.evaluate( QoQExec, intersection ); + compareResult = QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ); return compareResult == 1 || compareResult == 0; case LESSTHAN : - return Compare.invoke( left.evaluate( QoQExec, intersection ), right.evaluate( QoQExec, intersection ), true ) == -1; + leftValue = left.evaluate( QoQExec, intersection ); + rightValue = right.evaluate( QoQExec, intersection ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) == -1; case LESSTHANOREQUAL : - compareResult = Compare.invoke( left.evaluate( QoQExec, intersection ), right.evaluate( QoQExec, intersection ), true ); + leftValue = left.evaluate( QoQExec, intersection ); + rightValue = right.evaluate( QoQExec, intersection ); + compareResult = QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ); return compareResult == -1 || compareResult == 0; case MINUS : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -235,7 +243,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return leftNum - rightNum; case BITWISE_AND : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -243,7 +251,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return leftNum.intValue() & rightNum.intValue(); case BITWISE_OR : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -251,7 +259,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return leftNum.intValue() | rightNum.intValue(); case BITWISE_XOR : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -259,7 +267,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return leftNum.intValue() ^ rightNum.intValue(); case MODULO : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -270,7 +278,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { } return leftNum % rightNum; case MULTIPLY : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumber( left, QoQExec, intersection ); rightNum = evalAsNumber( right, QoQExec, intersection ); if ( leftNum == null || rightNum == null ) { @@ -280,9 +288,9 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { case NOTEQUAL : leftValue = left.evaluate( QoQExec, intersection ); rightValue = right.evaluate( QoQExec, intersection ); - return !EqualsEquals.invoke( leftValue, rightValue, true ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) != 0; case AND : - ensureBooleanOperands( QoQExec ); + // ensureBooleanOperands( QoQExec ); leftValue = left.evaluate( QoQExec, intersection ); // Short circuit, don't eval right if left is false if ( ( Boolean ) leftValue ) { @@ -291,7 +299,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { return false; } case OR : - ensureBooleanOperands( QoQExec ); + // ensureBooleanOperands( QoQExec ); if ( ( Boolean ) left.evaluate( QoQExec, intersection ) ) { return true; } @@ -342,7 +350,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse // Implement each binary operator switch ( operator ) { case DIVIDE : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -355,19 +363,23 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse case EQUAL : leftValue = left.evaluateAggregate( QoQExec, intersections ); rightValue = right.evaluateAggregate( QoQExec, intersections ); - return EqualsEquals.invoke( leftValue, rightValue, true ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) == 0; case GREATERTHAN : - return Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ) == 1; + return QoQCompare.invoke( left.getType( QoQExec ), left.evaluateAggregate( QoQExec, intersections ), + right.evaluateAggregate( QoQExec, intersections ) ) == 1; case GREATERTHANOREQUAL : - compareResult = Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ); + compareResult = QoQCompare.invoke( left.getType( QoQExec ), left.evaluateAggregate( QoQExec, intersections ), + right.evaluateAggregate( QoQExec, intersections ) ); return compareResult == 1 || compareResult == 0; case LESSTHAN : - return Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ) == -1; + return QoQCompare.invoke( left.getType( QoQExec ), left.evaluateAggregate( QoQExec, intersections ), + right.evaluateAggregate( QoQExec, intersections ) ) == -1; case LESSTHANOREQUAL : - compareResult = Compare.invoke( left.evaluateAggregate( QoQExec, intersections ), right.evaluateAggregate( QoQExec, intersections ), true ); + compareResult = QoQCompare.invoke( left.getType( QoQExec ), left.evaluateAggregate( QoQExec, intersections ), + right.evaluateAggregate( QoQExec, intersections ) ); return compareResult == -1 || compareResult == 0; case MINUS : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -375,7 +387,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse } return leftNum - rightNum; case BITWISE_AND : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -383,7 +395,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse } return leftNum.intValue() & rightNum.intValue(); case BITWISE_OR : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -391,7 +403,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse } return leftNum.intValue() | rightNum.intValue(); case BITWISE_XOR : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -399,7 +411,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse } return leftNum.intValue() ^ rightNum.intValue(); case MODULO : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -410,7 +422,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse } return leftNum % rightNum; case MULTIPLY : - ensureNumericOperands( QoQExec ); + // ensureNumericOperands( QoQExec ); leftNum = evalAsNumberAggregate( left, QoQExec, intersections ); rightNum = evalAsNumberAggregate( right, QoQExec, intersections ); if ( leftNum == null || rightNum == null ) { @@ -420,9 +432,9 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse case NOTEQUAL : leftValue = left.evaluateAggregate( QoQExec, intersections ); rightValue = right.evaluateAggregate( QoQExec, intersections ); - return !EqualsEquals.invoke( leftValue, rightValue, true ); + return QoQCompare.invoke( left.getType( QoQExec ), leftValue, rightValue ) != 0; case AND : - ensureBooleanOperands( QoQExec ); + // ensureBooleanOperands( QoQExec ); leftValue = left.evaluateAggregate( QoQExec, intersections ); // Short circuit, don't eval right if left is false if ( ( Boolean ) leftValue ) { @@ -431,7 +443,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse return false; } case OR : - ensureBooleanOperands( QoQExec ); + // ensureBooleanOperands( QoQExec ); if ( ( Boolean ) left.evaluateAggregate( QoQExec, intersections ) ) { return true; } @@ -543,6 +555,8 @@ private Double evalAsNumber( SQLExpression expression, QoQSelectExecution QoQExe } else { throw new BoxRuntimeException( "Cannot string as a number: [" + s + "]" ); } + } else { + nValue = NumberCaster.cast( value ); } if ( nValue == null ) { return null; @@ -572,6 +586,8 @@ private Double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecuti } else { throw new BoxRuntimeException( "Cannot string as a number: [" + s + "]" ); } + } else { + nValue = NumberCaster.cast( value ); } if ( nValue == null ) { return null; @@ -581,8 +597,7 @@ private Double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecuti @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java index bfccf4bfa..8eee1741a 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInOperation.java @@ -22,8 +22,8 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQCompare; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; -import ortus.boxlang.runtime.operators.EqualsEquals; /** * Abstract Node class representing SQL IN operation @@ -112,7 +112,7 @@ public boolean isBoolean( QoQSelectExecution QoQExec ) { public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Object value = expression.evaluate( QoQExec, intersection ); for ( SQLExpression v : values ) { - if ( EqualsEquals.invoke( value, v.evaluate( QoQExec, intersection ), true ) ) { + if ( QoQCompare.invoke( expression.getType( QoQExec ), value, v.evaluate( QoQExec, intersection ) ) == 0 ) { return !not; } } @@ -131,8 +131,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java index 752175ddc..8cbb90eae 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLInSubQueryOperation.java @@ -23,8 +23,8 @@ import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; import ortus.boxlang.compiler.ast.visitor.ReplacingBoxVisitor; import ortus.boxlang.compiler.ast.visitor.VoidBoxVisitor; +import ortus.boxlang.runtime.jdbc.qoq.QoQCompare; import ortus.boxlang.runtime.jdbc.qoq.QoQSelectExecution; -import ortus.boxlang.runtime.operators.EqualsEquals; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Query; @@ -118,7 +118,7 @@ public Object evaluate( QoQSelectExecution QoQExec, int[] intersection ) { Query subResult = QoQExec.getIndepententSubQuery( subquery ); Key firstAndOnlyColName = subResult.getColumns().keySet().iterator().next(); for ( Object v : subResult.getColumnData( firstAndOnlyColName ) ) { - if ( EqualsEquals.invoke( value, v, true ) ) { + if ( QoQCompare.invoke( expression.getType( QoQExec ), value, v ) == 0 ) { return !not; } } @@ -137,8 +137,7 @@ public Object evaluateAggregate( QoQSelectExecution QoQExec, List interse @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java index fa8d54f58..d9f663353 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java +++ b/src/main/java/ortus/boxlang/compiler/ast/sql/select/expression/operation/SQLUnaryOperation.java @@ -239,8 +239,7 @@ private Double evalAsNumberAggregate( SQLExpression expression, QoQSelectExecuti @Override public void accept( VoidBoxVisitor v ) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException( "Unimplemented method 'accept'" ); + v.visit( this ); } @Override diff --git a/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java b/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java index 7ffb699ae..8cd29063c 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java @@ -60,6 +60,25 @@ import ortus.boxlang.compiler.ast.expression.BoxStructType; import ortus.boxlang.compiler.ast.expression.BoxTernaryOperation; import ortus.boxlang.compiler.ast.expression.BoxUnaryOperation; +import ortus.boxlang.compiler.ast.sql.select.SQLTableVariable; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCase; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCaseWhenThen; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCountFunction; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLFunction; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLOrderBy; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLParam; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLParenthesis; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLStarExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLBooleanLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNullLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNumberLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLStringLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBetweenOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBinaryOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInSubQueryOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperation; import ortus.boxlang.compiler.ast.statement.BoxAnnotation; import ortus.boxlang.compiler.ast.statement.BoxArgumentDeclaration; import ortus.boxlang.compiler.ast.statement.BoxAssert; @@ -1738,4 +1757,213 @@ public void visit( BoxFunctionalMemberAccess node ) { printPostComments( node ); } + // SQL AST Nodes + + public void visit( SQLBooleanLiteral node ) { + printPreComments( node ); + print( String.valueOf( node.getValue() ) ); + printPostComments( node ); + } + + public void visit( SQLNullLiteral node ) { + printPreComments( node ); + print( "null" ); + printPostComments( node ); + } + + public void visit( SQLNumberLiteral node ) { + printPreComments( node ); + print( String.valueOf( node.getValue() ) ); + printPostComments( node ); + } + + public void visit( SQLStringLiteral node ) { + printPreComments( node ); + print( "'" ); + print( node.getValue().replace( "'", "''" ) ); + print( "'" ); + printPostComments( node ); + } + + public void visit( SQLBetweenOperation node ) { + printPreComments( node ); + node.getExpression().accept( this ); + if ( node.isNot() ) { + print( " not" ); + } + print( " between " ); + node.getLeft().accept( this ); + print( " and " ); + node.getRight().accept( this ); + printPostComments( node ); + } + + public void visit( SQLBinaryOperation node ) { + printPreComments( node ); + node.getLeft().accept( this ); + print( " " ); + print( node.getOperator().getSymbol() ); + print( " " ); + node.getRight().accept( this ); + printPostComments( node ); + } + + public void visit( SQLInOperation node ) { + printPreComments( node ); + node.getExpression().accept( this ); + if ( node.isNot() ) { + print( " not" ); + } + print( " in (" ); + int size = node.getValues().size(); + if ( size > 0 ) { + print( " " ); + } + for ( int i = 0; i < size; i++ ) { + node.getValues().get( i ).accept( this ); + if ( i < size - 1 ) { + print( ", " ); + } + } + if ( size > 0 ) { + print( " " ); + } + print( ")" ); + printPostComments( node ); + } + + public void visit( SQLInSubQueryOperation node ) { + printPreComments( node ); + node.getExpression().accept( this ); + if ( node.isNot() ) { + print( " not" ); + } + print( " in (" ); + node.getSubQuery().accept( this ); + print( ")" ); + printPostComments( node ); + } + + public void visit( SQLUnaryOperation node ) { + printPreComments( node ); + print( node.getOperator().getSymbol() ); + node.getExpression().accept( this ); + printPostComments( node ); + } + + public void visit( SQLCase node ) { + printPreComments( node ); + print( "case" ); + if ( node.getInputExpression() != null ) { + print( " " ); + node.getInputExpression().accept( this ); + } + increaseIndent(); + for ( var whenThen : node.getWhenThens() ) { + whenThen.accept( this ); + } + if ( node.getElseExpression() != null ) { + print( " else " ); + node.getElseExpression().accept( this ); + } + decreaseIndent(); + print( " end" ); + printPostComments( node ); + } + + public void visit( SQLCaseWhenThen node ) { + printPreComments( node ); + print( " when " ); + node.getWhenExpression().accept( this ); + print( " then " ); + node.getThenExpression().accept( this ); + printPostComments( node ); + } + + public void visit( SQLColumn node ) { + printPreComments( node ); + // TODO, actually track in the SQLColumn node what we had for the original table reference + if ( node.getTable() != null && node.getTable() instanceof SQLTableVariable stv ) { + print( stv.getAlias() != null ? stv.getAlias().getName() : stv.getName().getName() ); + print( "." ); + } + print( node.getName().getName() ); + printPostComments( node ); + } + + public void visit( SQLCountFunction node ) { + printPreComments( node ); + print( "count( " ); + if ( node.isDistinct() ) { + print( "distinct " ); + } + node.getArguments().get( 0 ).accept( this ); + print( " )" ); + printPostComments( node ); + } + + public void visit( SQLFunction node ) { + printPreComments( node ); + print( node.getName().getName() ); + print( "(" ); + int size = node.getArguments().size(); + if ( size > 0 ) { + print( " " ); + } + for ( int i = 0; i < size; i++ ) { + node.getArguments().get( i ).accept( this ); + if ( i < size - 1 ) { + print( ", " ); + } + } + if ( size > 0 ) { + print( " " ); + } + print( ")" ); + printPostComments( node ); + } + + public void visit( SQLOrderBy node ) { + printPreComments( node ); + node.getExpression().accept( this ); + if ( !node.isAscending() ) { + print( " desc" ); + } + printPostComments( node ); + } + + public void visit( SQLParam node ) { + printPreComments( node ); + if ( node.getName() != null ) { + print( ":" ); + print( node.getName() ); + } else { + // I need this to be a unique for each ordered param + print( "? /* position: " ); + print( String.valueOf( node.getPosition() ) ); + print( " */" ); + + } + printPostComments( node ); + } + + public void visit( SQLParenthesis node ) { + printPreComments( node ); + print( "( " ); + node.getExpression().accept( this ); + print( " )" ); + printPostComments( node ); + } + + public void visit( SQLStarExpression node ) { + printPreComments( node ); + // TODO, actually track in the SQLColumn node what we had for the original table reference + if ( node.getTable() != null && node.getTable() instanceof SQLTableVariable stv ) { + print( stv.getAlias() != null ? stv.getAlias().getName() : stv.getName().getName() ); + print( "." ); + } + print( "*" ); + printPostComments( node ); + } + } diff --git a/src/main/java/ortus/boxlang/compiler/ast/visitor/VoidBoxVisitor.java b/src/main/java/ortus/boxlang/compiler/ast/visitor/VoidBoxVisitor.java index 236272e42..efa1e146e 100644 --- a/src/main/java/ortus/boxlang/compiler/ast/visitor/VoidBoxVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/ast/visitor/VoidBoxVisitor.java @@ -55,6 +55,25 @@ import ortus.boxlang.compiler.ast.expression.BoxStructLiteral; import ortus.boxlang.compiler.ast.expression.BoxTernaryOperation; import ortus.boxlang.compiler.ast.expression.BoxUnaryOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCase; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCaseWhenThen; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLColumn; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLCountFunction; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLFunction; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLOrderBy; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLParam; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLParenthesis; +import ortus.boxlang.compiler.ast.sql.select.expression.SQLStarExpression; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLBooleanLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNullLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLNumberLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.literal.SQLStringLiteral; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBetweenOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLBinaryOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLInSubQueryOperation; +import ortus.boxlang.compiler.ast.sql.select.expression.operation.SQLUnaryOperation; import ortus.boxlang.compiler.ast.statement.BoxAnnotation; import ortus.boxlang.compiler.ast.statement.BoxArgumentDeclaration; import ortus.boxlang.compiler.ast.statement.BoxAssert; @@ -378,4 +397,82 @@ public void visit( BoxFunctionalMemberAccess node ) { visitChildren( node ); } + // SQL AST Nodes + + public void visit( SQLBooleanLiteral node ) { + visitChildren( node ); + } + + public void visit( SQLNullLiteral node ) { + visitChildren( node ); + } + + public void visit( SQLNumberLiteral node ) { + visitChildren( node ); + } + + public void visit( SQLStringLiteral node ) { + visitChildren( node ); + } + + public void visit( SQLBetweenOperation node ) { + visitChildren( node ); + } + + public void visit( SQLBinaryOperation node ) { + visitChildren( node ); + } + + public void visit( SQLInOperation node ) { + visitChildren( node ); + } + + public void visit( SQLInSubQueryOperation node ) { + visitChildren( node ); + } + + public void visit( SQLUnaryOperation node ) { + visitChildren( node ); + } + + public void visit( SQLCase node ) { + visitChildren( node ); + } + + public void visit( SQLCaseWhenThen node ) { + visitChildren( node ); + } + + public void visit( SQLColumn node ) { + visitChildren( node ); + } + + public void visit( SQLCountFunction node ) { + visitChildren( node ); + } + + public void visit( SQLExpression node ) { + visitChildren( node ); + } + + public void visit( SQLFunction node ) { + visitChildren( node ); + } + + public void visit( SQLOrderBy node ) { + visitChildren( node ); + } + + public void visit( SQLParam node ) { + visitChildren( node ); + } + + public void visit( SQLParenthesis node ) { + visitChildren( node ); + } + + public void visit( SQLStarExpression node ) { + visitChildren( node ); + } + } diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java index ead79d298..6f06e4cfe 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/SQLVisitor.java @@ -215,7 +215,7 @@ public SQLSelect visitSelect_core( Select_coreContext ctx ) { for ( int i = 1; i < ctx.table_or_subquery().size(); i++ ) { var tableCtx = ctx.table_or_subquery().get( i ); SQLTable joinTable = ( SQLTable ) visit( tableCtx ); - joins.add( new SQLJoin( SQLJoinType.FULL, joinTable, null, tools.getPosition( tableCtx ), tools.getSourceText( tableCtx ) ) ); + joins.add( new SQLJoin( SQLJoinType.CROSS, joinTable, null, tools.getPosition( tableCtx ), tools.getSourceText( tableCtx ) ) ); } } } else if ( ctx.join_clause() != null ) { diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQCompare.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQCompare.java new file mode 100644 index 000000000..d9a1081b8 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQCompare.java @@ -0,0 +1,106 @@ + +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package ortus.boxlang.runtime.jdbc.qoq; + +import ortus.boxlang.runtime.operators.Compare; +import ortus.boxlang.runtime.types.QueryColumnType; +import ortus.boxlang.runtime.types.exceptions.DatabaseException; + +/** + * I handle executing query of queries + */ +public class QoQCompare { + + /** + * Perform a compare optimized for query types. + * We're assuming both inputs are the same type for now + * + * @param type the type of the expressions being compared + * @param left the left value + * @param right the right value + * + * @return + */ + public static int invoke( QueryColumnType type, Object left, Object right ) { + Integer result = null; + + // This code may suffer from cast exceptions due to us assuming the type of the column + // Add defensive checks to this as neccessary. We are purposefully not just letting the casters sort it + // since we want raw performance based on the types we SHOULD already be able to know from the query + // System.out.println( "QoQCompare.java: invoke: type: " + type + ", left: " + left + ", right: " + right ); + try { + if ( left == null && right == null ) { + result = 0; + } else if ( left == null ) { + result = -1; + } else if ( right == null ) { + result = 1; + } else if ( type == QueryColumnType.VARCHAR || type == QueryColumnType.CHAR ) { + result = left.toString().compareToIgnoreCase( right.toString() ); + } else if ( type == QueryColumnType.BIGINT || type == QueryColumnType.DECIMAL || type == QueryColumnType.DOUBLE + || type == QueryColumnType.INTEGER ) { + if ( left instanceof Double ld && right instanceof Double rd ) { + result = ld.compareTo( rd ); + } else if ( left instanceof Integer li && right instanceof Integer ri ) { + result = li.compareTo( ri ); + } else if ( left instanceof Long ll && right instanceof Long rl ) { + result = ll.compareTo( rl ); + } else if ( left instanceof Number ln && right instanceof Number rn ) { + result = Double.compare( ln.doubleValue(), rn.doubleValue() ); + } + } else if ( type == QueryColumnType.BIT || type == QueryColumnType.BOOLEAN ) { + // Account for an int or a boolean + Boolean bLeft = null; + Boolean bRight = null; + if ( left instanceof Boolean bl ) { + bLeft = bl; + } else if ( left instanceof Number nl ) { + bLeft = nl.intValue() == 1; + } + if ( bLeft != null ) { + if ( right instanceof Boolean br ) { + bRight = br; + } else if ( right instanceof Number nr ) { + bRight = nr.intValue() == 1; + } + if ( bRight != null ) { + result = bLeft.compareTo( bRight ); + } + } + + } else if ( type == QueryColumnType.DATE || type == QueryColumnType.TIME || type == QueryColumnType.TIMESTAMP ) { + // Dates SHOULD get picked up by the casters + result = Compare.invoke( left, right ); + // System.out.println( "compare.invoke1" ); + } else { + // All other types, we'll just let the casters sort it out + result = Compare.invoke( left, right ); + // System.out.println( "compare.invoke2" ); + } + + // If the casting didn't work for our strict comparisons above, we'll let the casters sort it out + if ( result == null ) { + result = Compare.invoke( left, right ); + // System.out.println( "compare.invoke3" ); + } + } catch ( ClassCastException e ) { + String leftType = left == null ? "null" : left.getClass().getName(); + String rightType = right == null ? "null" : right.getClass().getName(); + throw new DatabaseException( "SQL Comparison -- unexpected data [" + leftType + "/" + rightType + "] in column of type " + type.toString(), e ); + } + return result; + } +} diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index cb8117e0b..ab6c09df8 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -15,12 +15,14 @@ */ package ortus.boxlang.runtime.jdbc.qoq; -import java.sql.SQLException; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; import java.util.stream.Stream; import ortus.boxlang.compiler.ast.sql.SQLNode; @@ -41,7 +43,6 @@ import ortus.boxlang.runtime.dynamic.ExpressionInterpreter; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.interop.DynamicObject; -import ortus.boxlang.runtime.operators.Compare; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Query; @@ -130,19 +131,10 @@ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatem } } - // TODO: Implement a sort that doesn't turn the query into a list of structs and back again + // Apply our sort if ( QoQStmtExec.getOrderByColumns() != null ) { - target.sort( ( row1, row2 ) -> { - var orderBys = QoQStmtExec.getOrderByColumns(); - for ( var orderBy : orderBys ) { - var name = orderBy.name; - int result = Compare.invoke( row1.get( name ), row2.get( name ) ); - if ( result != 0 ) { - return orderBy.ascending ? result : -result; - } - } - return 0; - } ); + sort( target, QoQStmtExec.getOrderByColumns() ); + // These were just here for sorting. Nuke them now. if ( QoQStmtExec.getAdditionalColumns() != null ) { for ( Key key : QoQStmtExec.getAdditionalColumns() ) { @@ -151,18 +143,7 @@ public static Query executeSelectStatement( IBoxContext context, SQLSelectStatem } } - // This is the maxRows in the query options. It takes priority. - Long overallSelectLimit; - try { - overallSelectLimit = statement.getLargeMaxRows(); - } catch ( SQLException e ) { - throw new DatabaseException( "Error getting max rows from statement", e ); - } - // If that wasn't set, use the limit clause AFTER the order by (which could apply at the end of a union) - if ( overallSelectLimit == -1 ) { - overallSelectLimit = selectStatement.getLimitValue(); - } - + Long overallSelectLimit = QoQStmtExec.getOverallSelectLimit(); // If we have a limit for the final select, apply it here. if ( overallSelectLimit > -1 ) { target.truncate( overallSelectLimit ); @@ -192,6 +173,13 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta SQLExpression where = select.getWhere(); boolean hasTable = select.getTable() != null; Long thisSelectLimit = select.getLimitValue(); + // If we can early limit, and there are no unions, apply the select statement limit here, if smaller + if ( canEarlyLimit && QoQStmtExec.getSelectStatement().getUnions() == null ) { + Long overallSelectLimit = QoQStmtExec.getOverallSelectLimit(); + if ( overallSelectLimit > -1 && ( thisSelectLimit == -1 || overallSelectLimit < thisSelectLimit ) ) { + thisSelectLimit = overallSelectLimit; + } + } if ( hasTable ) { // Tables are added in the order of their index, which represents the encounter-order in the SQL @@ -220,11 +208,12 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta } // Create empty query object to hold result - Query target = buildTargetQuery( QoQExec ); + Query target; // If there are no tables, and we are just selecting out literal values, we can just add the row and return // This code path ignores the where clause and top/limit. While technically vaid, it is not a common use case. if ( !hasTable ) { + target = buildTargetQuery( QoQExec, 1 ); Object[] values = new Object[ resultColumns.size() ]; for ( Key key : resultColumns.keySet() ) { SQLResultColumn resultColumn = resultColumns.get( key ).resultColumn; @@ -234,6 +223,8 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta target.addRow( values ); return target; } + // initial size is sum of all rows in all tables + target = buildTargetQuery( QoQExec, tableLookup.values().stream().mapToInt( Query::size ).sum() ); // We have one or more tables, so build our stream of intersections, processing our joins as needed Stream intersections = QoQIntersectionGenerator.createIntersectionStream( QoQExec ); @@ -248,7 +239,6 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta if ( canEarlyLimit && !select.isDistinct() && thisSelectLimit > -1 ) { intersections = intersections.limit( thisSelectLimit ); } - if ( select.hasAggregateResult() || select.getGroupBys() != null ) { target = executeAggregateSelect( QoQExec, target, intersections ); } else { @@ -264,6 +254,7 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta } finalTarget.addRow( values ); } ); + ( ( ArrayList ) target.getData() ).trimToSize(); } // Apply distinct to the final result set @@ -318,6 +309,7 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t values[ resultColumn.getOrdinalPosition() - 1 ] = value; } target.addRow( values ); + ( ( ArrayList ) target.getData() ).trimToSize(); return target; } @@ -341,6 +333,7 @@ private static Query executeAggregateSelect( QoQSelectExecution QoQExec, Query t } target.addRow( values ); } ); + ( ( ArrayList ) target.getData() ).trimToSize(); return target; } @@ -364,22 +357,45 @@ private static void unionAll( Query target, Query unionQuery ) { * @param unionQuery the query to union */ private static void deDupeQuery( Query target ) { - Set seen = new HashSet<>(); - // loop over rows, build partition key out of all values - for ( int i = 0; i < target.size(); i++ ) { - StringBuilder sb = new StringBuilder(); - Object[] row = target.getRow( i ); + Map seen = new ConcurrentHashMap<>(); + Object concurrentHashMapsAreDumb = new Object(); + Object[][] newData = new Object[ target.size() ][]; + Object[][] oldData = target.getData().toArray( new Object[ 0 ][] ); + AtomicInteger newSize = new AtomicInteger( 0 ); + int estimatedRowSize = target.getColumns().size() * 15; + + // Stream over indices and directly remove rows if they're duplicates + IntStream stream = IntStream.range( 0, target.size() ); + if ( target.size() > 100 ) { + stream = stream.parallel(); + } + stream.forEach( i -> { + StringBuilder sb = new StringBuilder( estimatedRowSize ); + Object[] row = oldData[ i ]; for ( Object value : row ) { sb.append( value ); } String key = sb.toString(); - if ( !seen.contains( key ) ) { - seen.add( key ); - } else { - target.deleteRow( i ); - i--; + if ( seen.putIfAbsent( key, concurrentHashMapsAreDumb ) == null ) { + newData[ newSize.getAndIncrement() ] = row; } - } + } ); + + Object[][] finalData = new Object[ newSize.get() ][]; + System.arraycopy( newData, 0, finalData, 0, newSize.get() ); + target.setData( new ArrayList<>( Arrays.asList( finalData ) ) ); + } + + private static void sort( Query target, List orderBys ) { + target.sortData( ( row1, row2 ) -> { + for ( NameAndDirection orderBy : orderBys ) { + int result = QoQCompare.invoke( orderBy.type, row1[ orderBy.position ], row2[ orderBy.position ] ); + if ( result != 0 ) { + return orderBy.ascending ? result : -result; + } + } + return 0; + } ); } /** @@ -389,9 +405,9 @@ private static void deDupeQuery( Query target ) { * * @return the target query */ - private static Query buildTargetQuery( QoQSelectExecution QoQExec ) { + private static Query buildTargetQuery( QoQSelectExecution QoQExec, int initialSize ) { Map resultColumns = QoQExec.getResultColumns(); - Query target = new Query(); + Query target = new Query( initialSize ); for ( Key key : resultColumns.keySet() ) { target.addColumn( key, resultColumns.get( key ).type ); } @@ -420,18 +436,18 @@ private static Query getSourceQuery( IBoxContext context, QoQSelectStatementExec * Represent a result column with a runtime type * TODO: We may not need this since the expression can tell us the type directly */ - public record TypedResultColumn( QueryColumnType type, SQLResultColumn resultColumn ) { + public record TypedResultColumn( QueryColumnType type, int position, SQLResultColumn resultColumn ) { - public static TypedResultColumn of( QueryColumnType type, SQLResultColumn resultColumn ) { - return new TypedResultColumn( type, resultColumn ); + public static TypedResultColumn of( QueryColumnType type, int position, SQLResultColumn resultColumn ) { + return new TypedResultColumn( type, position, resultColumn ); } } // Represent the name and order of an order by statement. This is calculated at runtime since the actual column names may be based on a * - public record NameAndDirection( Key name, boolean ascending ) { + public record NameAndDirection( Key name, QueryColumnType type, int position, boolean ascending ) { - public static NameAndDirection of( Key name, boolean ascending ) { - return new NameAndDirection( name, ascending ); + public static NameAndDirection of( Key name, QueryColumnType type, int position, boolean ascending ) { + return new NameAndDirection( name, type, position, ascending ); } } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java index 2b128dbab..991563fb3 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQIntersectionGenerator.java @@ -15,6 +15,7 @@ package ortus.boxlang.runtime.jdbc.qoq; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -75,6 +76,7 @@ public static Stream createIntersectionStream( QoQSelectExecution QoQExec if ( totalCombinations > 50 ) { theStream = theStream.parallel(); } + return theStream; } @@ -219,7 +221,7 @@ private static Stream handleFullOuterJoin( Stream theStream, Query } ); // Combine LEFT JOIN and RIGHT JOIN results and remove duplicates using a Set - Set> seen = new HashSet<>(); + Set> seen = Collections.synchronizedSet( new HashSet<>() ); return Stream.concat( leftJoinStream, rightJoinStream ) .filter( arr -> seen.add( Arrays.stream( arr ).boxed().collect( Collectors.toList() ) ) ); // Remove duplicates } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java index a27522cb2..19dfb83d7 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectExecution.java @@ -157,10 +157,11 @@ public Map calculateResultColumns( boolean firstSelect ) resultColumns.put( key, TypedResultColumn.of( thisTable.getColumns().get( key ).getType(), + resultColumns.size(), new SQLResultColumn( new SQLColumn( t, key.getName(), null, null ), null, - resultColumns.size() + 1, + resultColumns.size(), null, null ) @@ -169,9 +170,9 @@ public Map calculateResultColumns( boolean firstSelect ) } } ); // Non-star columns are named after the column, or given a column_0, column_1, etc name - } else { + } else if ( !resultColumns.containsKey( resultColumn.getResultColumnName() ) ) { resultColumns.put( resultColumn.getResultColumnName(), - TypedResultColumn.of( resultColumn.getExpression().getType( this ), resultColumn ) ); + TypedResultColumn.of( resultColumn.getExpression().getType( this ), resultColumns.size(), resultColumn ) ); } } setResultColumns( resultColumns ); @@ -212,7 +213,8 @@ public void calculateOrderBys() { return false; } ).findFirst(); if ( match.isPresent() ) { - orderByColumns.add( NameAndDirection.of( match.get().getKey(), orderBy.isAscending() ) ); + orderByColumns.add( + NameAndDirection.of( match.get().getKey(), match.get().getValue().type(), match.get().getValue().position(), orderBy.isAscending() ) ); continue; } } else if ( expr instanceof SQLNumberLiteral num ) { @@ -222,9 +224,29 @@ public void calculateOrderBys() { throw new DatabaseException( "The column index [" + index + "] in the order by clause is out of range as there are only " + numOriginalResulColumns + " column(s)." ); } - orderByColumns.add( NameAndDirection.of( resultColumns.keySet().toArray( new Key[ 0 ] )[ index - 1 ], orderBy.isAscending() ) ); + orderByColumns.add( + NameAndDirection.of( + resultColumns.keySet().toArray( new Key[ 0 ] )[ index - 1 ], + resultColumns.values().toArray( new TypedResultColumn[ 0 ] )[ index - 1 ].type(), + index - 1, + orderBy.isAscending() + ) + ); continue; + } else { + // Loop over result columns like above, but compare the tostring() representations to look for a match + var match = resultColumns.entrySet() + .stream() + .filter( rc -> expr.toString().equals( rc.getValue().resultColumn().getExpression().toString() ) ) + .findFirst(); + + if ( match.isPresent() ) { + orderByColumns.add( + NameAndDirection.of( match.get().getKey(), match.get().getValue().type(), match.get().getValue().position(), orderBy.isAscending() ) ); + continue; + } } + // TODO: This isn't quite right as a literal expression is technically OK in the order by of a union query, even though it's fairly useless. // We need the query sort to be rewritten to eval expressions on the fly for that to work however. Not worth addressing at the moment. if ( isUnion ) { @@ -237,10 +259,12 @@ public void calculateOrderBys() { // TODO: Figure out if this exact expression is already in the result set and use that // To do this, we need something like toString() implemented to compare two expressions for equivalence - Key newName = Key.of( "__order_by_column_" + additionalCounter++ ); + Key newName = Key.of( "__order_by_column_" + additionalCounter++ ); + int newColPos = resultColumns.size(); resultColumns.put( newName, - TypedResultColumn.of( QueryColumnType.OBJECT, new SQLResultColumn( expr, newName.getName(), resultColumns.size() + 1, null, null ) ) ); - orderByColumns.add( NameAndDirection.of( newName, orderBy.isAscending() ) ); + TypedResultColumn.of( QueryColumnType.OBJECT, newColPos, + new SQLResultColumn( expr, newName.getName(), newColPos, null, null ) ) ); + orderByColumns.add( NameAndDirection.of( newName, expr.getType( this ), newColPos, orderBy.isAscending() ) ); additionalColumns.add( newName ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java index 2c9406d23..c9c7fd829 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQSelectStatementExecution.java @@ -14,6 +14,7 @@ */ package ortus.boxlang.runtime.jdbc.qoq; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,6 +27,7 @@ import ortus.boxlang.runtime.jdbc.qoq.QoQPreparedStatement.ParamItem; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.exceptions.DatabaseException; /** * A wrapper class to hold together both the SQL AST being executed as well as the runtime values for a given execution of the query @@ -185,4 +187,27 @@ public QoQStatement getJDBCStatement() { return JDBCStatement; } + /** + * Get the overall select limit + * + * @return The overall select limit + */ + public Long getOverallSelectLimit() { + Long overallSelectLimit = -1L; + // This is the maxRows in the query options. + try { + overallSelectLimit = JDBCStatement.getLargeMaxRows(); + } catch ( SQLException e ) { + throw new DatabaseException( "Error getting max rows from statement", e ); + } + // If that wasn't set, use the limit clause AFTER the order by (which could apply at the end of a union) + if ( overallSelectLimit == -1 ) { + overallSelectLimit = selectStatement.getLimitValue(); + } else if ( selectStatement.getLimitValue() > -1 ) { + // If both are set, take the smaller of the two + overallSelectLimit = Math.min( selectStatement.getLimitValue(), overallSelectLimit ); + } + return overallSelectLimit; + } + } diff --git a/src/main/java/ortus/boxlang/runtime/operators/StringCompare.java b/src/main/java/ortus/boxlang/runtime/operators/StringCompare.java index ba26eb2b0..68ffe0858 100644 --- a/src/main/java/ortus/boxlang/runtime/operators/StringCompare.java +++ b/src/main/java/ortus/boxlang/runtime/operators/StringCompare.java @@ -101,7 +101,7 @@ public static Integer attempt( String left, String right, Boolean caseSensitive, .anyMatch( s -> s.codePoints().anyMatch( c -> c > 127 ) ); // if our locale is different than an EN locale use the Collator - if ( containsUnicode || ( !locale.equals( LocalizationUtil.COMMON_LOCALES.get( Key.of( "US" ) ) ) && !locale.equals( Locale.ENGLISH ) ) ) { + if ( containsUnicode || ( !locale.equals( LocalizationUtil.COMMON_LOCALES.get( Key.US ) ) && !locale.equals( Locale.ENGLISH ) ) ) { Collator collator = Collator.getInstance( locale ); return collator.getCollationKey( caseSensitive ? left.toString() : left.toString().toLowerCase( locale ) ) .compareTo( collator.getCollationKey( caseSensitive ? right.toString() : right.toString().toLowerCase( locale ) ) ); diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java index b68bda119..27ac94394 100644 --- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java +++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java @@ -737,6 +737,7 @@ public class Key implements Comparable, Serializable { public static final Key uploadAll = Key.of( "uploadAll" ); public static final Key URL = Key.of( "URL" ); public static final Key urlToken = Key.of( "urlToken" ); + public static final Key US = Key.of( "US" ); public static final Key useCache = Key.of( "useCache" ); public static final Key useCustomSerializer = Key.of( "useCustomSerializer" ); public static final Key useHighPrecisionMath = Key.of( "useHighPrecisionMath" ); diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index 24bd20cab..f685ec494 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -63,7 +64,11 @@ public class Query implements IType, IReferenceable, Collection, Serial * Query data as List of arrays */ // private List data = Collections.synchronizedList( new ArrayList() ); - private List data = new ArrayList(); + private List data; + + protected AtomicInteger size = new AtomicInteger( 0 ); + + private int actualSize = 0; /** * Map of column definitions @@ -95,16 +100,42 @@ public class Query implements IType, IReferenceable, Collection, Serial * * @param meta Struct of metadata, most likely JDBC metadata such as sql, cache parameters, etc. */ - public Query( IStruct meta ) { + public Query( IStruct meta, int initialSize ) { this.functionService = BoxRuntime.getInstance().getFunctionService(); this.metadata = meta == null ? new Struct( IStruct.TYPES.SORTED ) : meta; + if ( initialSize > 0 ) { + this.data = new ArrayList( initialSize ); + // add nulls and increment for each row + actualSize = initialSize; + for ( int i = 0; i < initialSize; i++ ) { + data.add( null ); + } + } else { + this.data = new ArrayList(); + } + } + + /** + * Create a new query with additional metadata + * + * @param meta Struct of metadata, most likely JDBC metadata such as sql, cache parameters, etc. + */ + public Query( IStruct meta ) { + this( meta, 0 ); } /** * Create a new query with a default (empty) metadata struct */ public Query() { - this( new Struct( IStruct.TYPES.SORTED ) ); + this( new Struct( IStruct.TYPES.SORTED ), 0 ); + } + + /** + * Create a new query with a default (empty) metadata struct + */ + public Query( int initialSize ) { + this( new Struct( IStruct.TYPES.SORTED ), initialSize ); } /** @@ -205,6 +236,12 @@ public List getData() { return data; } + public void setData( List data ) { + this.data = data; + size.set( data.size() ); + actualSize = data.size(); + } + /** * Add a column to the query, populated with nulls * @@ -240,10 +277,10 @@ public synchronized Query addColumn( Key name, QueryColumnType type, Object[] co } } columns.put( name, createQueryColumn( name, type, newColIndex ) ); - if ( !data.isEmpty() ) { + if ( size.get() > 0 ) { // loop over data and replace each array with a new array having an additional // null at the end - for ( int i = 0; i < data.size(); i++ ) { + for ( int i = 0; i < size.get(); i++ ) { Object[] row = data.get( i ); Object[] newRow = new Object[ row.length + 1 ]; System.arraycopy( row, 0, newRow, 0, row.length ); @@ -258,7 +295,7 @@ public synchronized Query addColumn( Key name, QueryColumnType type, Object[] co for ( Object columnDatum : columnData ) { Object[] row = new Object[ columns.size() ]; row[ newColIndex ] = columnDatum; - data.add( row ); + addRow( row ); } } return this; @@ -289,8 +326,8 @@ protected QueryColumn createQueryColumn( Key name, QueryColumnType type, int ind */ public Object[] getColumnData( Key name ) { int index = getColumn( name ).getIndex(); - Object[] columnData = new Object[ data.size() ]; - for ( int i = 0; i < data.size(); i++ ) { + Object[] columnData = new Object[ size.get() ]; + for ( int i = 0; i < size.get(); i++ ) { columnData[ i ] = data.get( i )[ index ]; } return columnData; @@ -394,9 +431,10 @@ public Query insertQueryAt( int position, Query target ) { } // Insert the rows - synchronized ( this ) { + synchronized ( data ) { for ( int i = 0; i < target.size(); i++ ) { data.add( position + i, target.getRow( i ) ); + size.incrementAndGet(); } } @@ -412,11 +450,19 @@ public Query insertQueryAt( int position, Query target ) { */ public int addRow( Object[] row ) { // TODO: validate types - int newRow; - synchronized ( this ) { - data.add( row ); - newRow = data.size(); + int newRow = size.incrementAndGet(); + if ( actualSize < newRow + 50 ) { + synchronized ( data ) { + if ( actualSize < newRow + 50 ) { + // Add 200 more rows with nulls + for ( int i = 0; i < 200; i++ ) { + data.add( null ); + } + actualSize = actualSize + 200; + } + } } + data.set( newRow - 1, row ); return newRow; } @@ -541,7 +587,9 @@ public void deleteColumn( Key name ) { */ public Query deleteRow( int index ) { validateRow( index ); + size.decrementAndGet(); data.remove( index ); + actualSize = data.size(); return this; } @@ -654,8 +702,8 @@ public Query setCell( Key columnName, int rowIndex, Object value ) { * @param index row index, 0-based */ public void validateRow( int index ) { - if ( index < 0 || index >= data.size() ) { - throw new BoxRuntimeException( "Row index " + index + " is out of bounds for query of size " + data.size() ); + if ( index < 0 || index >= size.get() ) { + throw new BoxRuntimeException( "Row index " + index + " is out of bounds for query of size " + size.get() ); } } @@ -705,17 +753,40 @@ public void sort( Comparator compareFunc ) { .collect( Collectors.toList() ); } + public void sortData( Comparator comparator ) { + Stream stream; + truncateInternal(); + if ( size() > 50 ) { + stream = getData().parallelStream(); + } else { + stream = getData().stream(); + } + this.data = stream.sorted( comparator ).collect( Collectors.toList() ); + } + /** * Truncate the query to a specific number of rows * This method does not lock the query and would allow other modifications or access while trimming the rows, whcih is not an atomic operation. */ public Query truncate( long rows ) { - rows = Math.max( 0, rows ); + synchronized ( data ) { + truncateInternal(); + rows = Math.max( 0, rows ); + // loop and remove all rows over the count + while ( size.get() > rows ) { + data.remove( size.decrementAndGet() - 1 ); + actualSize--; + } + return this; + } + } + + private void truncateInternal() { // loop and remove all rows over the count - while ( data.size() > rows ) { + while ( data.size() > size.get() ) { data.remove( data.size() - 1 ); } - return this; + actualSize = data.size(); } /*************************** @@ -723,7 +794,7 @@ public Query truncate( long rows ) { ****************************/ @Override public int size() { - return data.size(); + return size.get(); } @Override @@ -745,7 +816,7 @@ public Iterator iterator() { @Override public boolean hasNext() { - return index < data.size(); + return index < size.get(); } @Override @@ -759,12 +830,14 @@ public IStruct next() { @Override public Object[] toArray() { - return data.toArray(); + // return data as an array, but limit this to size.get + return data.subList( 0, size.get() ).toArray(); } @Override public T[] toArray( T[] a ) { - return data.toArray( a ); + // same as toArray + return data.subList( 0, size.get() ).toArray( a ); } /** @@ -789,7 +862,10 @@ public boolean add( IStruct row ) { @Override public boolean remove( Object o ) { synchronized ( data ) { - return data.remove( o ); + size.decrementAndGet(); + var result = data.remove( o ); + actualSize = data.size(); + return result; } } @@ -809,20 +885,32 @@ public boolean addAll( Collection rows ) { @Override public boolean removeAll( Collection c ) { synchronized ( data ) { - return data.removeAll( c ); + truncateInternal(); + boolean result = data.removeAll( c ); + size.set( data.size() ); + actualSize = data.size(); + return result; } } @Override public boolean retainAll( Collection c ) { synchronized ( data ) { - return data.retainAll( c ); + truncateInternal(); + boolean result = data.retainAll( c ); + size.set( data.size() ); + actualSize = data.size(); + return result; } } @Override public void clear() { - data.clear(); + synchronized ( data ) { + size.set( 0 ); + actualSize = 0; + data.clear(); + } } /*************************** @@ -887,7 +975,7 @@ public Object assign( IBoxContext context, Key name, Object value ) { public String asString() { StringBuilder sb = new StringBuilder(); sb.append( "[\n" ); - for ( int i = 0; i < data.size(); i++ ) { + for ( int i = 0; i < size.get(); i++ ) { if ( i > 0 ) { sb.append( ",\n" ); } @@ -910,7 +998,7 @@ public BoxMeta getBoxMeta() { * Returns a IntStream of the indexes */ public IntStream intStream() { - return IntStream.range( 0, data.size() ); + return IntStream.range( 0, size.get() ); } /** @@ -930,7 +1018,7 @@ public IStruct getMetaData() { this.metadata.putIfAbsent( Key.cacheProvider, null ); this.metadata.computeIfAbsent( Key.cacheTimeout, key -> Duration.ZERO ); this.metadata.computeIfAbsent( Key.cacheLastAccessTimeout, key -> Duration.ZERO ); - this.metadata.computeIfAbsent( Key.recordCount, key -> data.size() ); + this.metadata.computeIfAbsent( Key.recordCount, key -> size.get() ); this.metadata.computeIfAbsent( Key.columns, key -> this.getColumns() ); this.metadata.computeIfAbsent( Key.columnList, key -> this.getColumnList() ); this.metadata.computeIfAbsent( Key._HASHCODE, key -> this.hashCode() ); @@ -989,13 +1077,18 @@ public int computeHashCode( Set visited ) { return 0; } visited.add( this ); - int result = 1; + int result = 1; + int row = 1; for ( Object value : data.toArray() ) { + if ( row > size.get() ) { + break; + } if ( value instanceof IType ) { result = 31 * result + ( ( IType ) value ).computeHashCode( visited ); } else { result = 31 * result + ( value == null ? 0 : value.hashCode() ); } + row++; } return result; } @@ -1017,7 +1110,7 @@ public UnmodifiableQuery toUnmodifiable() { */ public Array asArrayOfStructs() { Array arr = new Array(); - for ( int i = 0; i < data.size(); i++ ) { + for ( int i = 0; i < size.get(); i++ ) { arr.add( getRowAsStruct( i ) ); } return arr; diff --git a/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java b/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java index 1c91450cd..ef2abfe16 100644 --- a/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java +++ b/src/main/java/ortus/boxlang/runtime/types/unmodifiable/UnmodifiableQuery.java @@ -72,9 +72,14 @@ public UnmodifiableQuery( Query query ) { super.addColumn( columnInfo.getValue().getName(), columnInfo.getValue().getType(), null ); } // then copy data + int i = 1; for ( Object[] row : query.getData() ) { + if ( i > query.size() ) { + break; + } Object[] duplicatedRow = row.clone(); super.addRow( duplicatedRow ); + i++; } } @@ -326,9 +331,14 @@ public Query toModifiable() { q.addColumn( columnInfo.getValue().getName(), columnInfo.getValue().getType(), null ); } // then copy data + int i = 1; for ( Object[] row : getData() ) { + if ( i > size.get() ) { + break; + } Object[] duplicatedRow = row.clone(); q.addRow( duplicatedRow ); + i++; } return q; From e11a02a9410149d8d612c9bad3f707eb4c89a82c Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 26 Dec 2024 18:37:45 -0600 Subject: [PATCH 060/161] BL-823 --- .../ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java | 2 +- src/main/java/ortus/boxlang/runtime/types/Query.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index ab6c09df8..e71e11ad6 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -174,7 +174,7 @@ public static Query executeSelect( IBoxContext context, SQLSelect select, QoQSta boolean hasTable = select.getTable() != null; Long thisSelectLimit = select.getLimitValue(); // If we can early limit, and there are no unions, apply the select statement limit here, if smaller - if ( canEarlyLimit && QoQStmtExec.getSelectStatement().getUnions() == null ) { + if ( canEarlyLimit && !select.isDistinct() && QoQStmtExec.getSelectStatement().getUnions() == null ) { Long overallSelectLimit = QoQStmtExec.getOverallSelectLimit(); if ( overallSelectLimit > -1 && ( thisSelectLimit == -1 || overallSelectLimit < thisSelectLimit ) ) { thisSelectLimit = overallSelectLimit; diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java index f685ec494..66a8cb14f 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Query.java +++ b/src/main/java/ortus/boxlang/runtime/types/Query.java @@ -233,6 +233,7 @@ public boolean hasColumn( Key name ) { * @return list of arrays of data */ public List getData() { + truncateInternal(); return data; } @@ -567,6 +568,7 @@ public int addRows( int rows ) { * @param name the name of the column to delete */ public void deleteColumn( Key name ) { + truncateInternal(); QueryColumn column = getColumn( name ); int index = column.getIndex(); columns.remove( name ); @@ -774,7 +776,7 @@ public Query truncate( long rows ) { rows = Math.max( 0, rows ); // loop and remove all rows over the count while ( size.get() > rows ) { - data.remove( size.decrementAndGet() - 1 ); + data.remove( size.decrementAndGet() ); actualSize--; } return this; @@ -799,7 +801,7 @@ public int size() { @Override public boolean isEmpty() { - return data.isEmpty(); + return size.get() == 0; } @Override From e027b389085775612359c0ab1061ee4c95812937 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 27 Dec 2024 13:00:12 -0600 Subject: [PATCH 061/161] BL-888 --- src/main/antlr/CFGrammar.g4 | 1 + .../compiler/toolchain/CFExpressionVisitor.java | 6 ++++-- src/test/java/TestCases/phase1/CoreLangTest.java | 13 +++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/antlr/CFGrammar.g4 b/src/main/antlr/CFGrammar.g4 index bb4d53de1..3f26ee5a2 100644 --- a/src/main/antlr/CFGrammar.g4 +++ b/src/main/antlr/CFGrammar.g4 @@ -479,6 +479,7 @@ structKey | INTEGER_LITERAL | ILLEGAL_IDENTIFIER | fqn + | SWITCH ; new: NEW preFix? (fqn | stringLiteral) LPAREN argumentList? RPAREN diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/CFExpressionVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/CFExpressionVisitor.java index 44488bd9a..2502dfb95 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/CFExpressionVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/CFExpressionVisitor.java @@ -1005,8 +1005,10 @@ public BoxExpression visitStructKey( StructKeyContext ctx ) { .orElseGet( () -> Optional.ofNullable( ctx.ILLEGAL_IDENTIFIER() ).map( fqn -> ( BoxExpression ) new BoxIdentifier( src, pos, src ) ) .orElseGet( () -> Optional.ofNullable( ctx.reservedOperators() ).map( resOp -> resOp.accept( this ) ) .orElseGet( () -> Optional.ofNullable( ctx.stringLiteral() ).map( str -> str.accept( this ) ) - .orElseGet( - () -> Optional.ofNullable( ctx.fqn() ).map( fqn -> fqn.accept( this ) ).orElse( new BoxIntegerLiteral( src, pos, src ) ) ) ) ) ); + .orElseGet( () -> Optional.ofNullable( ctx.SWITCH() ).map( swtch -> ( BoxExpression ) new BoxIdentifier( src, pos, src ) ) + .orElseGet( + () -> Optional.ofNullable( ctx.fqn() ).map( fqn -> fqn.accept( this ) ) + .orElse( new BoxIntegerLiteral( src, pos, src ) ) ) ) ) ) ); } @Override diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index eba94afac..5a9570fb5 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -4230,4 +4230,17 @@ public void testSoftRef() { assertThat( variables.get( result ) ).isInstanceOf( SoftReference.class ); } + @Test + public void testSwithStructKey() { + // @formatter:off + instance.executeSource( + """ + result = { switch: "" } + """, + context, BoxSourceType.CFSCRIPT ); + // @formatter:on + assertThat( variables.get( result ) ).isInstanceOf( IStruct.class ); + assertThat( variables.getAsStruct( result ) ).containsKey( Key.of( "switch" ) ); + } + } From 490cdd94b48686179f9197ee0b3d0d2e8fd63fad Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 27 Dec 2024 13:04:56 -0600 Subject: [PATCH 062/161] BL-886 --- .../ortus/boxlang/runtime/operators/Compare.java | 10 +++++++--- src/test/java/TestCases/phase1/CoreLangTest.java | 14 +++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/operators/Compare.java b/src/main/java/ortus/boxlang/runtime/operators/Compare.java index 140279292..62189eeeb 100644 --- a/src/main/java/ortus/boxlang/runtime/operators/Compare.java +++ b/src/main/java/ortus/boxlang/runtime/operators/Compare.java @@ -109,9 +109,13 @@ public static Integer attempt( Object left, Object right, Boolean caseSensitive, // Date comparison if ( DateTimeCaster.isKnownDateClass( left ) || DateTimeCaster.isKnownDateClass( right ) ) { - DateTime ref = DateTimeCaster.cast( left ); - DateTime target = DateTimeCaster.cast( right ); - return ref.compareTo( target ); + CastAttempt ref = DateTimeCaster.attempt( left ); + if ( ref.wasSuccessful() ) { + CastAttempt target = DateTimeCaster.attempt( right ); + if ( target.wasSuccessful() ) { + return ref.get().compareTo( target.get() ); + } + } } // Numeric comparison diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index 5a9570fb5..f87306542 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -4231,7 +4231,7 @@ public void testSoftRef() { } @Test - public void testSwithStructKey() { + public void testSwitchStructKey() { // @formatter:off instance.executeSource( """ @@ -4243,4 +4243,16 @@ public void testSwithStructKey() { assertThat( variables.getAsStruct( result ) ).containsKey( Key.of( "switch" ) ); } + @Test + public void testDateCOmpare() { + // @formatter:off + instance.executeSource( + """ + result = now() is "now" + """, + context ); + // @formatter:on + assertThat( variables.getAsBoolean( result ) ).isFalse(); + } + } From cdcb83fc2a0a6559e556b027e8b3dba81ccb96cb Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 27 Dec 2024 13:32:44 -0600 Subject: [PATCH 063/161] BL-885 --- .../expression/BoxStringConcatTransformer.java | 12 ++++++++++-- .../expression/BoxStringConcatTransformer.java | 5 +++++ src/test/java/TestCases/phase1/AssignmentTest.java | 13 +++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxStringConcatTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxStringConcatTransformer.java index 6acf512a1..ced011b21 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxStringConcatTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxStringConcatTransformer.java @@ -32,6 +32,7 @@ import ortus.boxlang.compiler.asmboxpiler.transformer.TransformerContext; import ortus.boxlang.compiler.ast.BoxNode; import ortus.boxlang.compiler.ast.expression.BoxStringConcat; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.operators.Concat; public class BoxStringConcatTransformer extends AbstractTransformer { @@ -44,8 +45,15 @@ public BoxStringConcatTransformer( Transpiler transpiler ) { public List transform( BoxNode node, TransformerContext context, ReturnValueContext returnContext ) throws IllegalStateException { BoxStringConcat interpolation = ( BoxStringConcat ) node; if ( interpolation.getValues().size() == 1 ) { - // TODO should this invoke StringCaster? - return transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.NONE, returnContext ); + List nodes = new ArrayList<>(); + nodes.addAll( transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.NONE, returnContext ) ); + nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, + Type.getInternalName( StringCaster.class ), + "cast", + Type.getMethodDescriptor( Type.getType( String.class ), Type.getType( Object.class ) ), + false ) ); + return AsmHelper.addLineNumberLabels( nodes, node ); + } else { List nodes = new ArrayList<>(); nodes.addAll( AsmHelper.array( Type.getType( Object.class ), interpolation.getValues(), diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java index c2ec592e9..16afd3baf 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java @@ -55,6 +55,11 @@ public Node transform( BoxNode node, TransformerContext context ) throws Illegal Node javaExpr; if ( interpolation.getValues().size() == 1 ) { javaExpr = transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.RIGHT ); + // force a string cast + NameExpr nameExpr = new NameExpr( "StringCaster" ); + MethodCallExpr methodCallExpr = new MethodCallExpr( nameExpr, "cast" ); + methodCallExpr.addArgument( ( Expression ) javaExpr ); + javaExpr = methodCallExpr; } else { List operands = interpolation.getValues() diff --git a/src/test/java/TestCases/phase1/AssignmentTest.java b/src/test/java/TestCases/phase1/AssignmentTest.java index c1a5bd82f..c4ae5b5e5 100644 --- a/src/test/java/TestCases/phase1/AssignmentTest.java +++ b/src/test/java/TestCases/phase1/AssignmentTest.java @@ -204,4 +204,17 @@ public void testQuotedAssignment() { assertThat( variables.get( Key.of( "result2" ) ) ).isEqualTo( "test2" ); } + @DisplayName( "quoted assignment 2" ) + @Test + public void testQuotedAssignment2() { + instance.executeSource( + """ + resultKey = "result"; + "#resultKey#" = "test"; + """, + context ); + assertThat( variables.get( result ) ).isEqualTo( "test" ); + + } + } From 7af1060984f8908ce229d7c2ebf66f502d0a370d Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 27 Dec 2024 13:43:05 -0600 Subject: [PATCH 064/161] BL-885 --- .../expression/BoxAssignmentTransformer.java | 9 ++++++++- .../expression/BoxStringConcatTransformer.java | 11 +---------- .../expression/BoxAssignmentTransformer.java | 2 +- .../expression/BoxStringConcatTransformer.java | 5 ----- .../boxlang/runtime/components/system/Loop.java | 12 +++++++----- src/main/java/ortus/boxlang/runtime/scopes/Key.java | 1 + 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAssignmentTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAssignmentTransformer.java index 3447266e1..41a4d75a5 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAssignmentTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAssignmentTransformer.java @@ -51,6 +51,7 @@ import ortus.boxlang.runtime.dynamic.ExpressionInterpreter; import ortus.boxlang.runtime.dynamic.IReferenceable; import ortus.boxlang.runtime.dynamic.Referencer; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.operators.Concat; import ortus.boxlang.runtime.operators.Divide; import ortus.boxlang.runtime.operators.Minus; @@ -116,7 +117,7 @@ public List transformEquals( BoxExpression left, List transformEquals( BoxExpression left, List transform( BoxNode node, TransformerContext context, ReturnValueContext returnContext ) throws IllegalStateException { BoxStringConcat interpolation = ( BoxStringConcat ) node; if ( interpolation.getValues().size() == 1 ) { - List nodes = new ArrayList<>(); - nodes.addAll( transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.NONE, returnContext ) ); - nodes.add( new MethodInsnNode( Opcodes.INVOKESTATIC, - Type.getInternalName( StringCaster.class ), - "cast", - Type.getMethodDescriptor( Type.getType( String.class ), Type.getType( Object.class ) ), - false ) ); - return AsmHelper.addLineNumberLabels( nodes, node ); - + return transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.NONE, returnContext ); } else { List nodes = new ArrayList<>(); nodes.addAll( AsmHelper.array( Type.getType( Object.class ), interpolation.getValues(), diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAssignmentTransformer.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAssignmentTransformer.java index 46612360b..5d742dc37 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAssignmentTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAssignmentTransformer.java @@ -129,7 +129,7 @@ public Node transformEquals( BoxExpression left, Expression jRight, BoxAssignmen template = """ ExpressionInterpreter.setVariable( ${contextName}, - ${left}, + StringCaster.cast( ${left} ), ${right} ) """; diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java index 16afd3baf..c2ec592e9 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxStringConcatTransformer.java @@ -55,11 +55,6 @@ public Node transform( BoxNode node, TransformerContext context ) throws Illegal Node javaExpr; if ( interpolation.getValues().size() == 1 ) { javaExpr = transpiler.transform( interpolation.getValues().get( 0 ), TransformerContext.RIGHT ); - // force a string cast - NameExpr nameExpr = new NameExpr( "StringCaster" ); - MethodCallExpr methodCallExpr = new MethodCallExpr( nameExpr, "cast" ); - methodCallExpr.addArgument( ( Expression ) javaExpr ); - javaExpr = methodCallExpr; } else { List operands = interpolation.getValues() diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java index e376e8464..2d62f7b53 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java @@ -67,10 +67,10 @@ public Loop() { new Attribute( Key.startRow, "integer", Set.of( Validator.min( 1 ) ) ), new Attribute( Key.endRow, "integer", Set.of( Validator.min( 1 ) ) ), new Attribute( Key.label, "string", Set.of( Validator.NON_EMPTY ) ), - new Attribute( Key.times, "integer", Set.of( Validator.min( 0 ) ) ) + new Attribute( Key.times, "integer", Set.of( Validator.min( 0 ) ) ), + new Attribute( Key.step, "number", 1 ) /** - * step * array * characters */ @@ -106,6 +106,7 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod Object queryOrName = attributes.get( Key.query ); String label = attributes.getAsString( Key.label ); Integer times = attributes.getAsInteger( Key.times ); + Number step = attributes.getAsNumber( Key.step ); if ( times != null ) { return _invokeTimes( context, times, item, index, body, executionState, label ); @@ -114,7 +115,7 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod return _invokeArray( context, array, item, index, body, executionState, label ); } if ( to != null && from != null ) { - return _invokeRange( context, from, to, index, body, executionState, label ); + return _invokeRange( context, from, to, step, index, body, executionState, label ); } if ( file != null ) { return _invokeFile( context, file, index, body, executionState, label ); @@ -233,9 +234,10 @@ private BodyResult _invokeFile( IBoxContext context, String file, String index, return DEFAULT_RETURN; } - private BodyResult _invokeRange( IBoxContext context, Double from, Double to, String index, ComponentBody body, IStruct executionState, String label ) { + private BodyResult _invokeRange( IBoxContext context, Double from, Double to, Number step, String index, ComponentBody body, IStruct executionState, + String label ) { // Loop over array, executing body every time - for ( int i = from.intValue(); i <= to.intValue(); i++ ) { + for ( int i = from.intValue(); i <= to.intValue(); i = i + step.intValue() ) { // Set the index and item variables ExpressionInterpreter.setVariable( context, index, i ); // Run the code inside of the output loop diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java index 27ac94394..8a08a147b 100644 --- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java +++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java @@ -71,6 +71,7 @@ public class Key implements Comparable, Serializable { public static final Key _NUMERIC = Key.of( "numeric" ); public static final Key _PACKAGE = Key.of( "package" ); public static final Key _QUERY = Key.of( "query" ); + public static final Key step = Key.of( "step" ); public static final Key _STRING = Key.of( "string" ); public static final Key _STRUCT = Key.of( "struct" ); public static final Key _super = Key.of( "super" ); From 8beb3ee9e02ccf4a340e3d04eaeea701f71810ce Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 27 Dec 2024 13:47:23 -0600 Subject: [PATCH 065/161] BL-887 --- .../runtime/components/system/LoopTest.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java index 6c484509d..d0b201814 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java @@ -318,4 +318,73 @@ public void testLoopArrayCollection() { assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "123" ); } + @Test + public void testLoopFromTo() { + instance.executeSource( + """ + + + + + """, + context, BoxSourceType.BOXTEMPLATE ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "12345678910" ); + } + + @Test + public void testLoopFromToStep() { + instance.executeSource( + """ + + + + + """, + context, BoxSourceType.BOXTEMPLATE ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "13579" ); + } + + @Test + public void testLoopFromToStepContens() { + instance.executeSource( + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #contensDecrypt(encr, "test")# + """, + context, BoxSourceType.CFTEMPLATE ); + } + } From 3863f6a6d881497bd701f6b7d27e7d00751463fa Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 05:11:13 -0500 Subject: [PATCH 066/161] JDBC - BL-876 - Place affected row count in query meta --- .../boxlang/runtime/jdbc/ExecutedQuery.java | 20 ++++++++++++++++++- .../bifs/global/jdbc/QueryExecuteTest.java | 17 ++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java index 26e33337f..cda94b20c 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java @@ -93,6 +93,7 @@ public ExecutedQuery( @Nonnull Query results, @Nullable Object generatedKey ) { public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery, @Nonnull Statement statement, long executionTime, boolean hasResults ) { Object generatedKey = null; Query results = null; + int recordCount = 0; if ( statement instanceof QoQStatement qs ) { results = qs.getQueryResult(); @@ -103,14 +104,30 @@ public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery throw new DatabaseException( e.getMessage(), e ); } } + if ( hasResults ) { + recordCount = results.size(); + } // Capture generated keys, if any. try { + if ( !hasResults ) { + try { + int affectedCount = statement.getUpdateCount(); + if ( affectedCount > -1 ) { + recordCount = affectedCount; + } + } catch ( SQLException t ) { + logger.error( "Error getting update count", t ); + } + } + // @TODO: Test this conditional around the result set... + // if( !hasResults ){} try ( ResultSet keys = statement.getGeneratedKeys() ) { if ( keys != null && keys.next() ) { generatedKey = keys.getObject( 1 ); } } catch ( SQLException e ) { + // @TODO: drop the message check, since it doesn't support alternate languages. if ( e.getMessage().contains( "The statement must be executed before any results can be obtained." ) ) { logger.info( "SQL Server threw an error when attempting to retrieve generated keys. Am ignoring the error - no action is required. Error : [{}]", @@ -138,7 +155,8 @@ public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery "cacheKey", pendingQuery.getCacheKey(), "sql", pendingQuery.getOriginalSql(), "sqlParameters", Array.fromList( pendingQuery.getParameterValues() ), - "executionTime", executionTime + "executionTime", executionTime, + "recordCount", recordCount ); if ( generatedKey != null ) { diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index ac9208106..98a7f2926 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -487,6 +487,23 @@ public void testMaxRows() { assertEquals( 1, query.size() ); } + @DisplayName( "DELETE queries put the affected recordCount in query meta" ) + @Test + public void testDeleteQueryResult() { + instance.executeSource( + """ + result = queryExecute( + "DELETE FROM developers WHERE id = :id", + { "id" : 1 }, + { "result" : "queryResults" } + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + IStruct query = variables.getAsStruct( Key.of( "queryResults" ) ); + assertEquals( 1, query.get( Key.recordCount ) ); + } + @DisplayName( "It closes connection on completion" ) @Test public void testConnectionClose() { From caeb08d86b4463343d04e64041f0babde0760011 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 05:42:02 -0500 Subject: [PATCH 067/161] JDBC - Add test for generated key query meta --- .../bifs/global/jdbc/BaseJDBCTest.java | 8 ++++++++ .../bifs/global/jdbc/QueryExecuteTest.java | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java index 9e1b71d52..aa293f24e 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java @@ -18,6 +18,7 @@ import ortus.boxlang.runtime.scopes.VariablesScope; import ortus.boxlang.runtime.services.DatasourceService; import ortus.boxlang.runtime.types.Struct; +import ortus.boxlang.runtime.types.exceptions.DatabaseException; import tools.JDBCTestUtils; public class BaseJDBCTest { @@ -56,6 +57,13 @@ public static void setUp() { ); datasourceService.register( mssqlName, mssqlDatasource ); JDBCTestUtils.ensureTestTableExists( mssqlDatasource, setUpContext ); + + try { + mssqlDatasource.execute( "DROP TABLE generatedKeyTest", setUpContext ); + mssqlDatasource.execute( "CREATE TABLE generatedKeyTest( id INT IDENTITY(1,1) PRIMARY KEY, name VARCHAR(155))", setUpContext ); + } catch ( DatabaseException e ) { + // Ignore the exception if the table already exists + } } if ( JDBCTestUtils.hasMySQLModule() ) { diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index 98a7f2926..635462d09 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIf; +import ortus.boxlang.runtime.dynamic.casters.DoubleCaster; import ortus.boxlang.runtime.dynamic.casters.StructCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Array; @@ -560,6 +561,24 @@ insert into developers (id, name) OUTPUT INSERTED.* assertEquals( "Jon", query.getRowAsStruct( 2 ).get( Key._NAME ) ); } + @EnabledIf( "tools.JDBCTestUtils#hasMSSQLModule" ) + @DisplayName( "It sets generatedKey in query meta" ) + @Test + public void testGeneratedKey() { + instance.executeStatement( + """ + queryExecute( + "INSERT INTO generatedKeyTest (name) VALUES ( 'Michael' )", + {}, + { "result": "variables.result", "datasource" : "MSSQLdatasource" } + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( IStruct.class ); + IStruct meta = variables.getAsStruct( result ); + assertThat( DoubleCaster.cast( meta.get( Key.generatedKey ), false ) ).isEqualTo( 1.0d ); + } + @DisplayName( "It can execute multiple statements in a single queryExecute() call like Lucee" ) @Test public void testMultipleStatements() { From bb7e8265e737fec00e5d8f0615d5d264a7a514be Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 05:48:54 -0500 Subject: [PATCH 068/161] JDBC - Tweak generated key handling to avoid erroring when pulling generated keys MSSQL loves to throw errors if a generated key doesn't exist. (Instead of simply returning null as other DB vendors do.) Since MSSQL also supports multiple languages, it's not easily possible to detect the reason for the error. All in all, we should log a warning and move on since the query DID complete successfully. Resolves BL-884. --- .../boxlang/runtime/jdbc/ExecutedQuery.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java index cda94b20c..a5a60aa1a 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java @@ -119,22 +119,19 @@ public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery } catch ( SQLException t ) { logger.error( "Error getting update count", t ); } - } - // @TODO: Test this conditional around the result set... - // if( !hasResults ){} - try ( ResultSet keys = statement.getGeneratedKeys() ) { - if ( keys != null && keys.next() ) { - generatedKey = keys.getObject( 1 ); - } - } catch ( SQLException e ) { - // @TODO: drop the message check, since it doesn't support alternate languages. - if ( e.getMessage().contains( "The statement must be executed before any results can be obtained." ) ) { - logger.info( - "SQL Server threw an error when attempting to retrieve generated keys. Am ignoring the error - no action is required. Error : [{}]", - e.getMessage() ); - } else { - // @TODO Add in more info to this - throw new DatabaseException( e.getMessage(), e ); + try ( ResultSet keys = statement.getGeneratedKeys() ) { + if ( keys != null && keys.next() ) { + generatedKey = keys.getObject( 1 ); + } + } catch ( SQLException e ) { + // @TODO: drop the message check, since it doesn't support alternate languages. + if ( e.getMessage().contains( "The statement must be executed before any results can be obtained." ) ) { + logger.info( + "SQL Server threw an error when attempting to retrieve generated keys. Am ignoring the error - no action is required. Error : [{}]", + e.getMessage() ); + } else { + logger.warn( "Error getting generated keys", e ); + } } } } catch ( NullPointerException e ) { From dd876eeeaf902fd4ce2529715fce88b845e6a4d7 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 06:25:50 -0500 Subject: [PATCH 069/161] JDBC - Fix mutating DEFAULT custom parameters on datasource configs Resolves BL-891. --- .../boxlang/runtime/config/segments/DatasourceConfig.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java index f81b179eb..457eaafd0 100644 --- a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java +++ b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java @@ -145,9 +145,7 @@ public class DatasourceConfig implements Comparable, IConfigSe "autoCommit", true, // Register mbeans or not. By default, this is true // However, if you are using JMX, you can set this to true to get some additional monitoring information - "registerMbeans", true, - // Prep the custom properties - "custom", new Struct() + "registerMbeans", true ); // List of keys to NOT set dynamically. All keys not in this list will use `addDataSourceProperty` to set the property and pass it to the JDBC driver. @@ -387,6 +385,9 @@ public DatasourceConfig processProperties( IStruct properties ) { } } ); + // Prep custom properties. + this.properties.putIfAbsent( Key.custom, new Struct() ); + // Merge defaults into the properties DEFAULTS .entrySet() From cac3a3513ddaa566b7295a7cd3adc648eca48786 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 06:45:34 -0500 Subject: [PATCH 070/161] JDBC - Fix 'custom' property struct for DatasourceConfig Use the constructor, thats what its there for! --- .../runtime/config/segments/DatasourceConfig.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java index 457eaafd0..69c8d7de1 100644 --- a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java +++ b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java @@ -188,6 +188,7 @@ public class DatasourceConfig implements Comparable, IConfigSe */ public DatasourceConfig() { // Default all things + this.properties.putIfAbsent( Key.custom, new Struct() ); } /** @@ -208,6 +209,7 @@ public DatasourceConfig( String name, IStruct properties ) { */ public DatasourceConfig( Key name, IStruct properties ) { this.name = name; + this.properties.putIfAbsent( Key.custom, new Struct() ); processProperties( properties ); } @@ -219,8 +221,7 @@ public DatasourceConfig( Key name, IStruct properties ) { * @param properties The datasource configuration properties. */ public DatasourceConfig( IStruct properties ) { - processProperties( properties ); - this.name = Key.of( "unnamed_" + UUID.randomUUID().toString() ); + this( Key.of( "unnamed_" + UUID.randomUUID().toString() ), properties ); } /** @@ -230,6 +231,7 @@ public DatasourceConfig( IStruct properties ) { */ public DatasourceConfig( Key name ) { this.name = name; + this.properties.putIfAbsent( Key.custom, new Struct() ); } /** @@ -385,9 +387,6 @@ public DatasourceConfig processProperties( IStruct properties ) { } } ); - // Prep custom properties. - this.properties.putIfAbsent( Key.custom, new Struct() ); - // Merge defaults into the properties DEFAULTS .entrySet() From 0e530c5e619a70a297e9f806387bd58972e000e0 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 07:37:40 -0500 Subject: [PATCH 071/161] JDBC - Implement list support for queryparam component Resolves[BL-893](https://ortussolutions.atlassian.net/browse/BL-893) --- .../runtime/components/jdbc/QueryParam.java | 21 ++++++++++++- .../boxlang/runtime/jdbc/PendingQuery.java | 7 ++++- .../runtime/components/jdbc/QueryTest.java | 30 ++++++++++++++----- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java index 06682cdab..5f48711fc 100644 --- a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java +++ b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java @@ -17,11 +17,15 @@ */ package ortus.boxlang.runtime.components.jdbc; +import java.util.stream.Collectors; + import ortus.boxlang.runtime.components.Attribute; import ortus.boxlang.runtime.components.BoxComponent; import ortus.boxlang.runtime.components.Component; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; @BoxComponent( allowsBody = false ) @@ -51,8 +55,23 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod } // Set our data into the Query component for it to use parentState.getAsArray( Key.queryParams ).add( attributes ); - context.writeToBuffer( "?", true ); + String tokenReplacement = "?"; + if ( attributes.containsKey( Key.list ) && BooleanCaster.cast( attributes.get( Key.list ) ) ) { + Object val = attributes.get( Key.value ); + if ( val instanceof String ) { + tokenReplacement = buildPlaceholderTokenList( Array.fromString( ( String ) val, attributes.getAsString( Key.separator ) ) ); + } else if ( val instanceof Array ) { + tokenReplacement = buildPlaceholderTokenList( ( Array ) val ); + } + } + context.writeToBuffer( tokenReplacement, true ); return DEFAULT_RETURN; } + private String buildPlaceholderTokenList( Array listValue ) { + return ( ( Array ) listValue ) + .stream() + .map( v -> "?" ) + .collect( Collectors.joining( "," ) ); + } } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index 1b1c831b6..d3df70fd5 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -30,6 +30,7 @@ import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.cache.providers.ICacheProvider; +import ortus.boxlang.runtime.components.jdbc.QueryParam; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.Attempt; import ortus.boxlang.runtime.dynamic.casters.ArrayCaster; @@ -262,7 +263,11 @@ private List buildParameterList( @Nonnull IStruct parameters ) { } /** - * Massage the SQL string in case of list parameters. + * Massage the SQL string, replacing named parameters (`:name`) with positional placeholders (`?`). + * + * Query params do not insert named params, they insert positional params, and they already take care of inserting the correct number of placeholder tokens. + * + * @see {@link QueryParam#_invoke} for queryparam placeholder token insertion */ private String massageSQL() { if ( parameters.isEmpty() ) { diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index 2af683cb2..2bec5f00c 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -28,7 +28,6 @@ import java.util.List; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -158,18 +157,35 @@ public void testArrayBindings() { assertEquals( "Developer", michael.get( "role" ) ); } - @Disabled( "Parsing error!" ) - @DisplayName( "It can execute a query with a list queryparam" ) + @DisplayName( "It can execute a query with an array queryparam" ) @Test - public void testListBindings() { + public void testListArrayBinding() { getInstance().executeSource( """ + - SELECT * FROM developers WHERE id IN + SELECT * FROM developers WHERE id IN () """, - context ); - assertThat( getVariables().get( result ) ).isInstanceOf( Query.class ); + context, + BoxSourceType.CFTEMPLATE ); + assertThat( getVariables().get( result ) ).isInstanceOf( ortus.boxlang.runtime.types.Query.class ); + ortus.boxlang.runtime.types.Query query = getVariables().getAsQuery( result ); + assertEquals( 3, query.size() ); + } + + @DisplayName( "It can execute a query with a list queryparam" ) + @Test + public void testListStringBinding() { + getInstance().executeSource( + """ + + SELECT * FROM developers WHERE role IN () + + """, + context, + BoxSourceType.CFTEMPLATE ); + assertThat( getVariables().get( result ) ).isInstanceOf( ortus.boxlang.runtime.types.Query.class ); ortus.boxlang.runtime.types.Query query = getVariables().getAsQuery( result ); assertEquals( 3, query.size() ); } From 8848351def9a709c085866774b4875b61ad07655 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Sat, 28 Dec 2024 08:03:15 -0500 Subject: [PATCH 072/161] JDBC - Implement 'params' attribute support for query component Resolves [BL-877](https://ortussolutions.atlassian.net/browse/BL-877) --- .../runtime/components/jdbc/Query.java | 10 +++++-- .../runtime/components/jdbc/QueryTest.java | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/components/jdbc/Query.java b/src/main/java/ortus/boxlang/runtime/components/jdbc/Query.java index ae3fcb2d7..5c21a8c4c 100644 --- a/src/main/java/ortus/boxlang/runtime/components/jdbc/Query.java +++ b/src/main/java/ortus/boxlang/runtime/components/jdbc/Query.java @@ -126,8 +126,14 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod return bodyResult; } - String sql = buffer.toString(); - Array bindings = executionState.getAsArray( Key.queryParams ); + String sql = buffer.toString(); + Object bindings = executionState.getAsArray( Key.queryParams ); + if ( attributes.containsKey( Key.params ) && attributes.get( Key.params ) != null ) { + if ( ( ( Array ) bindings ).size() > 0 ) { + throw new IllegalArgumentException( "Cannot specify both query parameters in the body and as an attribute." ); + } + bindings = attributes.get( Key.params ); + } PendingQuery pendingQuery = new PendingQuery( sql, bindings, options ); ExecutedQuery executedQuery; diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index 2bec5f00c..8f474e6a5 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -473,4 +473,33 @@ public void testQueryCaching() { assertThat( queryMeta4.getAsBoolean( Key.cached ) ).isEqualTo( false ); } + @DisplayName( "It can pass params as a struct in a query attribute" ) + @Test + public void testParamAttribute() { + getInstance().executeSource( + """ + + SELECT * FROM developers WHERE id = :id OR id = :id2 + + """, + getContext(), BoxSourceType.BOXTEMPLATE ); + assertThat( getVariables().get( result ) ).isInstanceOf( ortus.boxlang.runtime.types.Query.class ); + ortus.boxlang.runtime.types.Query query = getVariables().getAsQuery( result ); + assertEquals( 2, query.size() ); + } + + @DisplayName( "It throws if mutually exclusive param sources are used" ) + @Test + public void testParamAttributeThrow() { + IllegalArgumentException e = assertThrows( IllegalArgumentException.class, () -> getInstance().executeSource( + """ + + SELECT * FROM developers WHERE id = + + """, + getContext(), BoxSourceType.BOXTEMPLATE ) ); + + assertThat( e.getMessage() ).contains( "Cannot specify both query parameters in the body and as an attribute" ); + assertNull( getVariables().get( result ) ); + } } From 1a453cc3bf744dfecbe7c2076272e89b3ea62ee1 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 28 Dec 2024 12:27:46 -0600 Subject: [PATCH 073/161] BL-895 --- src/main/java/ortus/boxlang/compiler/parser/CFLexerCustom.java | 2 +- src/test/java/TestCases/phase1/CoreLangTest.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/compiler/parser/CFLexerCustom.java b/src/main/java/ortus/boxlang/compiler/parser/CFLexerCustom.java index 2972c4bb4..1317a1bbe 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/CFLexerCustom.java +++ b/src/main/java/ortus/boxlang/compiler/parser/CFLexerCustom.java @@ -239,7 +239,7 @@ public Token nextToken() { // reserved operators after a dot are just identifiers // foo.var // bar.GT() - if ( dotty && operatorWords.contains( nextToken.getType() ) ) { + if ( dotty && ( operatorWords.contains( nextToken.getType() ) || nextToken.getType() == CFLexer.SWITCH ) ) { ( ( CommonToken ) nextToken ).setType( IDENTIFIER ); // reserved operators (other than NOT) before an open parenthesis are just identifiers // LT() diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index f87306542..4f1c513fa 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -4236,11 +4236,14 @@ public void testSwitchStructKey() { instance.executeSource( """ result = { switch: "" } + result.switch = "brad" + result2 = result.switch; """, context, BoxSourceType.CFSCRIPT ); // @formatter:on assertThat( variables.get( result ) ).isInstanceOf( IStruct.class ); assertThat( variables.getAsStruct( result ) ).containsKey( Key.of( "switch" ) ); + assertThat( variables.get( Key.of( "result2" ) ) ).isEqualTo( "brad" ); } @Test From 5fa0843fd615158528e1d085a752ef378cf90c93 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 28 Dec 2024 14:04:14 -0600 Subject: [PATCH 074/161] BL-893 WIP --- .../runtime/components/jdbc/QueryParam.java | 4 +- .../boxlang/runtime/jdbc/PendingQuery.java | 26 +++++---- .../boxlang/runtime/jdbc/QueryParameter.java | 8 ++- .../ortus/boxlang/runtime/types/Array.java | 6 +- .../boxlang/runtime/util/RegexBuilder.java | 55 +++++++++++++++++++ .../{QoQLists2Test.cfc_ => QoQLists2Test.cfc} | 0 .../{QoQListsTest.cfc_ => QoQListsTest.cfc} | 0 7 files changed, 81 insertions(+), 18 deletions(-) rename src/test/java/external/specs/{QoQLists2Test.cfc_ => QoQLists2Test.cfc} (100%) rename src/test/java/external/specs/{QoQListsTest.cfc_ => QoQListsTest.cfc} (100%) diff --git a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java index 5f48711fc..44e17cbf0 100644 --- a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java +++ b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java @@ -58,8 +58,8 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod String tokenReplacement = "?"; if ( attributes.containsKey( Key.list ) && BooleanCaster.cast( attributes.get( Key.list ) ) ) { Object val = attributes.get( Key.value ); - if ( val instanceof String ) { - tokenReplacement = buildPlaceholderTokenList( Array.fromString( ( String ) val, attributes.getAsString( Key.separator ) ) ); + if ( val instanceof String sVal ) { + tokenReplacement = buildPlaceholderTokenList( Array.fromString( sVal, attributes.getAsString( Key.separator ) ) ); } else if ( val instanceof Array ) { tokenReplacement = buildPlaceholderTokenList( ( Array ) val ); } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index d3df70fd5..a38d62e0d 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -273,21 +273,25 @@ private String massageSQL() { if ( parameters.isEmpty() ) { return sql; } + // Replace the params as we go, so we can process positionally in the order they appear in the SQL string. + String tempPlaceholder = "____QUESTION_MARK____"; for ( QueryParameter p : parameters ) { + String sqlReplacement = tempPlaceholder; if ( p.isListParam() ) { - Array v = ( Array ) p.getValue(); - String sqlReplacement = v.stream().map( param -> "?" ).collect( Collectors.joining( "," ) ); - String placeholder = ":" + p.getName(); - - // Matcher parenPattern = Pattern.compile( "\\(\\s+" + placeholder + "\\s+\\)" ).matcher( sql ); - // if ( !parenPattern.find() ) { - // sql = sql.replaceFirst( placeholder, "(" + sqlReplacement + ")" ); - // } else { - sql = sql.replaceFirst( placeholder, sqlReplacement ); - // } + Array v = ( Array ) p.getValue(); + sqlReplacement = v.stream().map( param -> tempPlaceholder ).collect( Collectors.joining( "," ) ); + } + String placeholder = "\\?"; + if ( p.getName() != null ) { + placeholder = ":" + p.getName(); + // If this is a named param, replace all instances of it + sql = RegexBuilder.of( sql, placeholder ).replaceAllAndGet( sqlReplacement ); + } else { + // If it's a positional param, replace the first instance + sql = RegexBuilder.of( sql, placeholder ).replaceFirstAndGet( sqlReplacement ); } } - this.sql = RegexBuilder.of( sql, RegexBuilder.SQL_PARAMETER ).replaceAllAndGet( "?" ); + this.sql = RegexBuilder.of( sql, tempPlaceholder ).replaceAllAndGet( "?" ); return sql; } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index d3f1b2ceb..028768ad3 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -17,6 +17,7 @@ import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.dynamic.casters.StructCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Array; @@ -89,13 +90,18 @@ private QueryParameter( String name, IStruct param ) { // allow nulls and null this.isNullParam = BooleanCaster.cast( param.getOrDefault( Key.nulls, param.getOrDefault( Key.nulls2, false ) ) ); this.isListParam = BooleanCaster.cast( param.getOrDefault( Key.list, false ) ); + String separator = StringCaster.cast( param.getOrDefault( Key.separator, "," ) ); + // Allow a name key in the struct if passed as an array of structs + if ( name == null ) { + name = ( String ) param.getAsString( Key._NAME ); + } Object v = param.get( Key.value ); if ( this.isListParam ) { if ( v instanceof Array ) { // do nothing? } else { - v = ListUtil.asList( ( String ) v, ( String ) param.getOrDefault( Key.separator, "," ) ); + v = ListUtil.asList( ( String ) v, ( String ) param.getOrDefault( Key.separator, separator ) ); } } diff --git a/src/main/java/ortus/boxlang/runtime/types/Array.java b/src/main/java/ortus/boxlang/runtime/types/Array.java index 384aa6390..7ea621664 100644 --- a/src/main/java/ortus/boxlang/runtime/types/Array.java +++ b/src/main/java/ortus/boxlang/runtime/types/Array.java @@ -56,7 +56,7 @@ import ortus.boxlang.runtime.types.meta.IChangeListener; import ortus.boxlang.runtime.types.meta.IListenable; import ortus.boxlang.runtime.types.unmodifiable.UnmodifiableArray; -import ortus.boxlang.runtime.types.util.BLCollector; +import ortus.boxlang.runtime.types.util.ListUtil; import ortus.boxlang.runtime.util.RegexBuilder; /** @@ -182,9 +182,7 @@ public static Array fromString( String list, String delimiter ) { } // Split the string by comma and trim the values - return Arrays.stream( list.split( delimiter ) ) - .map( String::trim ) - .collect( BLCollector.toArray() ); + return ListUtil.asList( list, delimiter ); } /** diff --git a/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java b/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java index 2d0729763..7c4039cdf 100644 --- a/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java +++ b/src/main/java/ortus/boxlang/runtime/util/RegexBuilder.java @@ -257,6 +257,39 @@ public RegexMatcher replaceAll( Pattern pattern, String replacement ) { return this; } + /** + * Replace first occurrence of the pattern in the input string with the replacement string + * + * @param pattern The pattern to match against + * @param replacement The replacement string + * + * @return The input string with all occurrences of the pattern replaced with the replacement string + */ + public RegexMatcher replaceFirst( String replacement ) { + Objects.requireNonNull( replacement, "Replacement cannot be null" ); + this.input = this.pattern + .matcher( this.input ) + .replaceFirst( replacement ); + return this; + } + + /** + * Replace first occurrence of the pattern in the input string with the replacement string + * + * @param pattern The pattern to match against + * @param replacement The replacement string + * + * @return The input string with all occurrences of the pattern replaced with the replacement string + */ + public RegexMatcher replaceFirst( Pattern pattern, String replacement ) { + Objects.requireNonNull( pattern, "Pattern cannot be null" ); + Objects.requireNonNull( replacement, "Replacement cannot be null" ); + this.input = pattern + .matcher( this.input ) + .replaceFirst( replacement ); + return this; + } + /** * Replace all occurrences of the pattern in the input string with the replacement string * @@ -279,6 +312,28 @@ public String replaceAllAndGet( Pattern pattern, String replacement ) { return this.replaceAll( pattern, replacement ).get(); } + /** + * Replace first occurrence of the pattern in the input string with the replacement string + * + * @param replacement The replacement string + * + * @return The input string with all occurrences of the pattern replaced with the replacement string + */ + public String replaceFirstAndGet( Pattern pattern, String replacement ) { + return this.replaceFirst( pattern, replacement ).get(); + } + + /** + * Replace first occurrence of the pattern in the input string with the replacement string + * + * @param replacement The replacement string + * + * @return The input string with all occurrences of the pattern replaced with the replacement string + */ + public String replaceFirstAndGet( String replacement ) { + return this.replaceFirst( replacement ).get(); + } + /** * Get's the input string as modified by the matcher * diff --git a/src/test/java/external/specs/QoQLists2Test.cfc_ b/src/test/java/external/specs/QoQLists2Test.cfc similarity index 100% rename from src/test/java/external/specs/QoQLists2Test.cfc_ rename to src/test/java/external/specs/QoQLists2Test.cfc diff --git a/src/test/java/external/specs/QoQListsTest.cfc_ b/src/test/java/external/specs/QoQListsTest.cfc similarity index 100% rename from src/test/java/external/specs/QoQListsTest.cfc_ rename to src/test/java/external/specs/QoQListsTest.cfc From d519dda0a8593b808ecb1069a48adacaa639be96 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 28 Dec 2024 14:21:34 -0600 Subject: [PATCH 075/161] BL-893 these are expanded in pending query --- .../runtime/components/jdbc/QueryParam.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java index 44e17cbf0..efb65e32e 100644 --- a/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java +++ b/src/main/java/ortus/boxlang/runtime/components/jdbc/QueryParam.java @@ -17,15 +17,11 @@ */ package ortus.boxlang.runtime.components.jdbc; -import java.util.stream.Collectors; - import ortus.boxlang.runtime.components.Attribute; import ortus.boxlang.runtime.components.BoxComponent; import ortus.boxlang.runtime.components.Component; import ortus.boxlang.runtime.context.IBoxContext; -import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.scopes.Key; -import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; @BoxComponent( allowsBody = false ) @@ -56,22 +52,8 @@ public BodyResult _invoke( IBoxContext context, IStruct attributes, ComponentBod // Set our data into the Query component for it to use parentState.getAsArray( Key.queryParams ).add( attributes ); String tokenReplacement = "?"; - if ( attributes.containsKey( Key.list ) && BooleanCaster.cast( attributes.get( Key.list ) ) ) { - Object val = attributes.get( Key.value ); - if ( val instanceof String sVal ) { - tokenReplacement = buildPlaceholderTokenList( Array.fromString( sVal, attributes.getAsString( Key.separator ) ) ); - } else if ( val instanceof Array ) { - tokenReplacement = buildPlaceholderTokenList( ( Array ) val ); - } - } context.writeToBuffer( tokenReplacement, true ); return DEFAULT_RETURN; } - private String buildPlaceholderTokenList( Array listValue ) { - return ( ( Array ) listValue ) - .stream() - .map( v -> "?" ) - .collect( Collectors.joining( "," ) ); - } } From 4a7daab0a3f2e716bb4bad037215857acffffdc8 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 28 Dec 2024 14:37:57 -0600 Subject: [PATCH 076/161] BL-893 fix test --- .../java/ortus/boxlang/runtime/components/jdbc/QueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index 8f474e6a5..d7694e0c7 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -478,8 +478,8 @@ public void testQueryCaching() { public void testParamAttribute() { getInstance().executeSource( """ - - SELECT * FROM developers WHERE id = :id OR id = :id2 + + SELECT * FROM developers WHERE id = :id OR id = :again """, getContext(), BoxSourceType.BOXTEMPLATE ); From e7b71d473a539431040026fdd291f185ef08734e Mon Sep 17 00:00:00 2001 From: Michael Born Date: Mon, 30 Dec 2024 07:22:18 -0500 Subject: [PATCH 077/161] JDBC - Write additional query param tests; resolve bug with mix of list and non-list param types --- .../boxlang/runtime/jdbc/PendingQuery.java | 2 +- .../bifs/global/jdbc/QueryExecuteTest.java | 68 ++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index a38d62e0d..9839b4e4c 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -485,8 +485,8 @@ private void applyParameters( Statement statement, IBoxContext context ) throws } else { preparedStatement.setObject( parameterIndex, param.toSQLType( context ), param.getSqlTypeAsInt(), scaleOrLength ); } + parameterIndex++; } - parameterIndex++; } } } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index 635462d09..c3ac5d6ba 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -172,9 +172,9 @@ public void testListStringBindings() { assertEquals( 3, query.size() ); } - @DisplayName( "It can execute a query with an (array) list binding" ) + @DisplayName( "It can execute a query with a named (array) list binding" ) @Test - public void testListArrayBindings() { + public void testListArrayNamedBindings() { instance.executeSource( """ result = queryExecute( @@ -188,6 +188,70 @@ public void testListArrayBindings() { assertEquals( 3, query.size() ); } + @DisplayName( "It can execute a query with an (array) list binding" ) + @Test + public void testListArrayPositionalBinding() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id IN (?)", + [ { value: [77, 1, 42], list : true } ] + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 3, query.size() ); + } + + @DisplayName( "It can reuse named query parameters" ) + @Test + public void testParamReuse() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id = :id OR name=:id", + { id: 77 } + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 1, query.size() ); + } + + @DisplayName( "It can use various query params types (list and non-list) in a single SQL query" ) + @Test + public void testParameterTypeMix() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id = ? OR id IN (?)", + [ 77, { value: [1, 42], list : true } ] + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 3, query.size() ); + } + + @DisplayName( "It can grab a queryparam name from a parameter array" ) + @Test + public void testNamedParamInsideParamArray() { + instance.executeSource( + """ + result = queryExecute( + "SELECT * FROM developers WHERE id = :foo", + [ { name: "foo", value: "1" } ] + ); + """, + context ); + assertThat( variables.get( result ) ).isInstanceOf( Query.class ); + Query query = variables.getAsQuery( result ); + assertEquals( 1, query.size() ); + } + @DisplayName( "It can execute a query with struct bindings on the default datasource" ) @Test public void testStructBindings() { From 2d6853ab1ac55bb776abe53879e45a8b233833bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:09:56 +0000 Subject: [PATCH 078/161] Bump org.semver4j:semver4j from 5.4.1 to 5.5.0 Bumps [org.semver4j:semver4j](https://github.com/semver4j/semver4j) from 5.4.1 to 5.5.0. - [Release notes](https://github.com/semver4j/semver4j/releases) - [Commits](https://github.com/semver4j/semver4j/compare/v5.4.1...v5.5.0) --- updated-dependencies: - dependency-name: org.semver4j:semver4j dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3f9123928..6d8f7fe13 100644 --- a/build.gradle +++ b/build.gradle @@ -136,7 +136,7 @@ dependencies { // https://mvnrepository.com/artifact/org.ow2.asm/asm-util implementation 'org.ow2.asm:asm-util:9.7.1' // https://mvnrepository.com/artifact/org.semver4j/semver4j - implementation 'org.semver4j:semver4j:5.4.1' + implementation 'org.semver4j:semver4j:5.5.0' } From acf7675e32213f99bbe5c469b29ba663eba7ea27 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 30 Dec 2024 12:47:41 -0600 Subject: [PATCH 079/161] BL-889 --- .../dynamic/casters/StructCasterLoose.java | 13 +- .../interop/DynamicInteropService.java | 29 ++- .../ortus/boxlang/runtime/scopes/Key.java | 18 +- .../types/exceptions/AbortException.java | 2 +- .../types/exceptions/BoxIOException.java | 3 - .../types/exceptions/BoxLangException.java | 9 +- .../types/exceptions/BoxRuntimeException.java | 6 +- .../types/exceptions/CustomException.java | 6 +- .../types/exceptions/DatabaseException.java | 26 +-- .../types/exceptions/ExceptionUtil.java | 31 ++- .../types/exceptions/ExpressionException.java | 10 +- .../types/exceptions/LockException.java | 11 +- .../exceptions/MissingIncludeException.java | 6 +- .../types/exceptions/ParseException.java | 2 +- src/main/resources/dump/html/Null.bxm | 3 +- src/main/resources/dump/html/Throwable.bxm | 198 ++++-------------- .../TestCases/components/BoxTemplateTest.java | 20 +- .../TestCases/components/CFTemplateTest.java | 36 ++-- .../runtime/bifs/global/system/ThrowTest.java | 10 +- 19 files changed, 167 insertions(+), 272 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java index f8d24b950..7a1790e78 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java @@ -76,9 +76,12 @@ public static IStruct cast( Object object, Boolean fail ) { } object = DynamicObject.unWrap( object ); - IStruct result = StructCaster.cast( object, false ); - if ( result != null ) { - return result; + // Struct caster calls Exception Util, which calls us, so avoid a stack overflow here + if ( ! ( object instanceof Throwable ) ) { + IStruct result = StructCaster.cast( object, false ); + if ( result != null ) { + return result; + } } // If it's a random Java class, then turn it into a struct!! @@ -88,13 +91,13 @@ public static IStruct cast( Object object, Boolean fail ) { dynObject.getFieldsAsStream() .filter( field -> Modifier.isPublic( field.getModifiers() ) ) .forEach( field -> { - thisResult.put( field.getName(), dynObject.getField( field.getName() ) ); + thisResult.put( field.getName(), dynObject.getField( field.getName() ).get() ); } ); // also add fields for all public methods starting with "get" that take no arguments dynObject.getMethodNames( true ).forEach( methodName -> { Method m; if ( methodName.startsWith( "get" ) && Modifier.isPublic( ( m = dynObject.getMethod( methodName, true ) ).getModifiers() ) - && m.getParameterCount() == 0 ) { + && m.getParameterCount() == 0 && !methodName.equals( "getClass" ) ) { thisResult.put( methodName.substring( 3 ), dynObject.invoke( BoxRuntime.getInstance().getRuntimeContext(), methodName ) ); } } ); diff --git a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java index 346be769c..bb0fec70b 100644 --- a/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java +++ b/src/main/java/ortus/boxlang/runtime/interop/DynamicInteropService.java @@ -72,6 +72,7 @@ import ortus.boxlang.runtime.types.exceptions.AbstractClassException; import ortus.boxlang.runtime.types.exceptions.BoxLangException; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; +import ortus.boxlang.runtime.types.exceptions.CustomException; import ortus.boxlang.runtime.types.exceptions.ExceptionUtil; import ortus.boxlang.runtime.types.exceptions.KeyNotFoundException; import ortus.boxlang.runtime.types.exceptions.NoConstructorException; @@ -123,14 +124,18 @@ public class DynamicInteropService { * -------------------------------------------------------------------------- */ + /** + * These keys are always available on all throwables. If the key is not found, then we return an empty string + */ private static final Set exceptionKeys = new HashSet<>( Arrays.asList( - BoxLangException.messageKey, - BoxLangException.detailKey, - BoxLangException.typeKey, - BoxLangException.tagContextKey, - BoxRuntimeException.ExtendedInfoKey, + Key.message, + Key.detail, + Key.type, + Key.tagContext, + Key.extendedinfo, Key.stackTrace, - Key.cause + Key.cause, + Key.errorcode ) ); /** @@ -1702,7 +1707,7 @@ public static Object dereference( IBoxContext context, Class targetClass, Obj return arr[ index - 1 ]; } else if ( targetInstance instanceof Throwable t && exceptionKeys.contains( name ) ) { // Throwable.message always delegates through to the message field - if ( name.equals( BoxLangException.messageKey ) ) { + if ( name.equals( Key.message ) ) { return t.getMessage(); } else if ( name.equals( Key.cause ) ) { return t.getCause(); @@ -1711,14 +1716,16 @@ public static Object dereference( IBoxContext context, Class targetClass, Obj PrintWriter pw = new PrintWriter( sw ); t.printStackTrace( pw ); return sw.toString(); - } else if ( name.equals( BoxLangException.tagContextKey ) ) { + } else if ( name.equals( Key.tagContext ) ) { return ExceptionUtil.buildTagContext( t ); } else if ( targetInstance instanceof BoxLangException ble ) { - if ( name.equals( BoxLangException.detailKey ) ) { + if ( name.equals( Key.detail ) ) { return ble.getDetail(); - } else if ( name.equals( BoxLangException.typeKey ) ) { + } else if ( name.equals( Key.type ) ) { return ble.getType(); - } else if ( ble instanceof BoxRuntimeException bre && name.equals( BoxRuntimeException.ExtendedInfoKey ) ) { + } else if ( targetInstance instanceof CustomException ce && name.equals( Key.errorcode ) ) { + return ce.getErrorCode(); + } else if ( ble instanceof BoxRuntimeException bre && name.equals( Key.extendedinfo ) ) { return bre.getExtendedInfo(); } else { return ""; diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java index 8a08a147b..d0490dabb 100644 --- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java +++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java @@ -71,7 +71,6 @@ public class Key implements Comparable, Serializable { public static final Key _NUMERIC = Key.of( "numeric" ); public static final Key _PACKAGE = Key.of( "package" ); public static final Key _QUERY = Key.of( "query" ); - public static final Key step = Key.of( "step" ); public static final Key _STRING = Key.of( "string" ); public static final Key _STRUCT = Key.of( "struct" ); public static final Key _super = Key.of( "super" ); @@ -148,9 +147,9 @@ public class Key implements Comparable, Serializable { public static final Key caller = Key.of( "caller" ); public static final Key canonicalize = Key.of( "canonicalize" ); public static final Key caseSensitive = Key.of( "caseSensitive" ); + public static final Key cast = Key.of( "cast" ); public static final Key category = Key.of( "category" ); public static final Key cause = Key.of( "cause" ); - public static final Key cast = Key.of( "cast" ); public static final Key cert_cookie = Key.of( "cert_cookie" ); public static final Key cert_flags = Key.of( "cert_flags" ); public static final Key cert_issuer = Key.of( "cert_issuer" ); @@ -200,8 +199,8 @@ public class Key implements Comparable, Serializable { public static final Key context = Key.of( "context" ); public static final Key context_path = Key.of( "context_path" ); public static final Key contextual = Key.of( "contextual" ); - public static final Key convert = Key.of( "convert" ); public static final Key conversionType = Key.of( "conversionType" ); + public static final Key convert = Key.of( "convert" ); public static final Key cookies = Key.of( "cookies" ); public static final Key copy = Key.of( "copy" ); public static final Key count = Key.of( "count" ); @@ -297,6 +296,7 @@ public class Key implements Comparable, Serializable { public static final Key entryPaths = Key.of( "entryPaths" ); public static final Key environment = Key.of( "environment" ); public static final Key equals = Key.of( "equals" ); + public static final Key errNumber = Key.of( "errNumber" ); public static final Key error = Key.of( "error" ); public static final Key errorcode = Key.of( "errorcode" ); public static final Key errorDetail = Key.of( "errorDetail" ); @@ -443,6 +443,9 @@ public class Key implements Comparable, Serializable { public static final Key local_host = Key.of( "local_host" ); public static final Key locale = Key.of( "locale" ); public static final Key localeSensitive = Key.of( "localeSensitive" ); + public static final Key localizedMessage = Key.of( "localizedMessage" ); + public static final Key lockName = Key.of( "lockName" ); + public static final Key lockOperation = Key.of( "lockOperation" ); public static final Key log = Key.of( "log" ); public static final Key logger = Key.of( "logger" ); public static final Key loggers = Key.of( "loggers" ); @@ -474,6 +477,7 @@ public class Key implements Comparable, Serializable { public static final Key min = Key.of( "min" ); public static final Key minute = Key.of( "minute" ); public static final Key minutes = Key.of( "minutes" ); + public static final Key missingFileName = Key.of( "missingFileName" ); public static final Key missingMethodArguments = Key.of( "missingMethodArguments" ); public static final Key missingMethodName = Key.of( "missingMethodName" ); public static final Key missingTemplate = Key.of( "missingTemplate" ); @@ -497,6 +501,7 @@ public class Key implements Comparable, Serializable { public static final Key nameAsKey = Key.of( "nameAsKey" ); public static final Key nameconflict = Key.of( "nameconflict" ); public static final Key namespace = Key.of( "namespace" ); + public static final Key nativeErrorCode = Key.of( "nativeErrorCode" ); public static final Key newBuffer = Key.of( "newBuffer" ); public static final Key newBuilder = Key.of( "newBuilder" ); public static final Key newDelimiter = Key.of( "newDelimiter" ); @@ -573,6 +578,7 @@ public class Key implements Comparable, Serializable { public static final Key query_string = Key.of( "query_string" ); public static final Key query1 = Key.of( "query1" ); public static final Key query2 = Key.of( "query2" ); + public static final Key queryError = Key.of( "queryError" ); public static final Key queryFormat = Key.of( "queryFormat" ); public static final Key queryParams = Key.of( "queryParams" ); public static final Key queryTimeout = Key.of( "queryTimeout" ); @@ -665,10 +671,10 @@ public class Key implements Comparable, Serializable { public static final Key source = Key.of( "source" ); public static final Key sql = Key.of( "sql" ); public static final Key sqlParameters = Key.of( "sqlParameters" ); + public static final Key SQLState = Key.of( "SQLState" ); public static final Key sqltype = Key.of( "sqltype" ); public static final Key stackTrace = Key.of( "stackTrace" ); public static final Key start = Key.of( "start" ); - public static final Key statusPrinterOnLoad = Key.of( "statusPrinterOnLoad" ); public static final Key startRow = Key.of( "startRow" ); public static final Key startTicks = Key.of( "startTicks" ); public static final Key startTime = Key.of( "startTime" ); @@ -678,7 +684,9 @@ public class Key implements Comparable, Serializable { public static final Key status_code = Key.of( "status_code" ); public static final Key status_text = Key.of( "status_text" ); public static final Key statusCode = Key.of( "statusCode" ); + public static final Key statusPrinterOnLoad = Key.of( "statusPrinterOnLoad" ); public static final Key statusText = Key.of( "statusText" ); + public static final Key step = Key.of( "step" ); public static final Key storedproc = Key.of( "storedproc" ); public static final Key stream = Key.of( "stream" ); public static final Key strict = Key.of( "strict" ); @@ -698,6 +706,7 @@ public class Key implements Comparable, Serializable { public static final Key substring1 = Key.of( "substring1" ); public static final Key substringMatch = Key.of( "substringMatch" ); public static final Key suffix = Key.of( "suffix" ); + public static final Key suppressed = Key.of( "suppressed" ); public static final Key suppressWhiteSpace = Key.of( "suppressWhiteSpace" ); public static final Key system = Key.of( "system" ); public static final Key systemExecute = Key.of( "systemExecute" ); @@ -763,6 +772,7 @@ public class Key implements Comparable, Serializable { public static final Key wddx = Key.of( "wddx" ); public static final Key web_server_api = Key.of( "web_server_api" ); public static final Key webURL = Key.of( "webURL" ); + public static final Key where = Key.of( "where" ); public static final Key whitespaceCompressionEnabled = Key.of( "whitespaceCompressionEnabled" ); public static final Key workstation = Key.of( "workstation" ); public static final Key write = Key.of( "write" ); diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/AbortException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/AbortException.java index 6bcfc82c9..c2645e743 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/AbortException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/AbortException.java @@ -26,7 +26,7 @@ */ public class AbortException extends RuntimeException { - public String type = "request"; + protected String type = "request"; /** * Constructor diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxIOException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxIOException.java index c6112c993..dbeba3630 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxIOException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxIOException.java @@ -20,8 +20,6 @@ import java.io.IOException; import java.nio.file.FileSystemException; -import ortus.boxlang.runtime.scopes.Key; - /** * This exception is thrown when an IO operation fails. * @@ -29,7 +27,6 @@ */ public class BoxIOException extends BoxRuntimeException { - public static final Key ErrorCodeKey = Key.errorcode; private static final String ACCESS_DENIED = "AccessDeniedException"; private static final String ATOMIC_MOVE_DENIED = "AtomicMoveNotSupportedException"; private static final String FILE_ALREADY_EXISTS = "FileAlreadyExistsException"; diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxLangException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxLangException.java index c6e4e88ee..384488815 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxLangException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxLangException.java @@ -27,16 +27,11 @@ */ public abstract class BoxLangException extends RuntimeException { - public static final Key messageKey = Key.of( "message" ); - public static final Key detailKey = Key.of( "detail" ); - public static final Key typeKey = Key.of( "type" ); - public static final Key tagContextKey = Key.of( "tagContext" ); - /** * Detailed message from the BoxLang parser or specified in a throw component. When the exception is generated by BoxLang (and not throw), the * message can contain HTML formatting and can help determine which component threw the exception. */ - public String detail = ""; + protected String detail = ""; /** * Type: Exception type, as specified in catch. * @@ -54,7 +49,7 @@ public abstract class BoxLangException extends RuntimeException { * The type must ALWAYS be set for a BoxLangException by the superclass extending this base class. This will also * ensure the type string matches the exception class as well. */ - public String type = null; + protected String type = null; /** * Constructor diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxRuntimeException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxRuntimeException.java index 4762d3c51..e8de0a477 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxRuntimeException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxRuntimeException.java @@ -27,12 +27,10 @@ */ public class BoxRuntimeException extends BoxLangException { - public static final Key ExtendedInfoKey = Key.extendedinfo; - /** * Custom error message; information that the default exception handler does not display. */ - public Object extendedInfo = ""; + protected Object extendedInfo = ""; /** * Constructor @@ -110,7 +108,7 @@ public Object getExtendedInfo() { public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( ExtendedInfoKey, this.extendedInfo ); + result.put( Key.extendedinfo, this.extendedInfo ); return result; } } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/CustomException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/CustomException.java index aba217397..184c0fb08 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/CustomException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/CustomException.java @@ -27,12 +27,10 @@ */ public class CustomException extends BoxRuntimeException { - public static final Key ErrorCodeKey = Key.of( "errorCode" ); - /** * Applies to type = "custom". String error code. */ - public String errorCode = ""; + protected String errorCode = ""; /** * Constructor @@ -106,7 +104,7 @@ public String getErrorCode() { @Override public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( ErrorCodeKey, errorCode ); + result.put( Key.errorcode, errorCode ); return result; } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/DatabaseException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/DatabaseException.java index cb9562c2f..7410ec846 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/DatabaseException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/DatabaseException.java @@ -25,34 +25,28 @@ */ public class DatabaseException extends BoxLangException { - public static final Key NativeErrorCodeKey = Key.of( "nativeErrorCode" ); - public static final Key SQLStateKey = Key.of( "SQLState" ); - public static final Key SqlKey = Key.of( "SQL" ); - public static final Key queryErrorKey = Key.of( "queryError" ); - public static final Key whereKey = Key.of( "where" ); - /** * Native error code associated with exception. Database drivers typically provide error codes to diagnose failing * database operations. Default value is -1. */ - public String nativeErrorCode = ""; + protected String nativeErrorCode = ""; /** * SQLState associated with exception. Database drivers typically provide error codes to help diagnose failing database * operations. Default value is 1. */ - public String SQLState = ""; + protected String SQLState = ""; /** * The SQL statement sent to the data source. */ - public String SQL = ""; + protected String SQL = ""; /** * The error message as reported by the database driver. */ - public String queryError = ""; + protected String queryError = ""; /** * If the query uses the queryparam component, query parameter name-value pairs. */ - public String where = ""; + protected String where = ""; /** * Constructor @@ -155,11 +149,11 @@ public String getWhere() { public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( NativeErrorCodeKey, nativeErrorCode ); - result.put( SQLStateKey, SQLState ); - result.put( SqlKey, SQL ); - result.put( queryErrorKey, queryError ); - result.put( whereKey, where ); + result.put( Key.nativeErrorCode, nativeErrorCode ); + result.put( Key.SQLState, SQLState ); + result.put( Key.sql, SQL ); + result.put( Key.queryError, queryError ); + result.put( Key.where, where ); return result; } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java index b6f5b3d22..67a5f52e6 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java @@ -38,6 +38,7 @@ import ortus.boxlang.compiler.ast.SourceFile; import ortus.boxlang.compiler.javaboxpiler.JavaBoxpiler; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.dynamic.casters.StructCasterLoose; import ortus.boxlang.runtime.dynamic.casters.ThrowableCaster; import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.operators.InstanceOf; @@ -450,15 +451,29 @@ public static IStruct throwableToStruct( Throwable target ) { if ( target == null ) { return null; } - IStruct result = Struct.of( - Key.message, target.getMessage(), - Key.stackTrace, ExceptionUtil.getStackTraceAsString( target ), - Key.tagContext, ExceptionUtil.buildTagContext( target ), - Key.cause, throwableToStruct( target.getCause() ) - ); - if ( target instanceof BoxLangException ble ) { - result.addAll( ble.dataAsStruct() ); + + // All getter methods will be called on the target, which will get all custom fields + // regardless of what the actual class is + IStruct result = StructCasterLoose.cast( target ); + result.put( Key.tagContext, ExceptionUtil.buildTagContext( target ) ); + result.put( Key.stackTrace, ExceptionUtil.getStackTraceAsString( target ) ); + result.put( Key.cause, throwableToStruct( target.getCause() ) ); + + // Ensure we have a type field + if ( !result.containsKey( Key.type ) ) { + result.put( Key.type, target.getClass().getName() ); + } + + if ( result.containsKey( Key.suppressed ) && result.get( Key.suppressed ) instanceof Array sa && sa.isEmpty() ) { + result.remove( Key.suppressed ); } + if ( result.containsKey( Key.cause ) && result.get( Key.cause ) == null ) { + result.remove( Key.cause ); + } + if ( result.containsKey( Key.localizedMessage ) ) { + result.remove( Key.localizedMessage ); + } + return result; } } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExpressionException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExpressionException.java index d8da7e918..455dabbe0 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExpressionException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExpressionException.java @@ -27,14 +27,12 @@ */ public class ExpressionException extends BoxRuntimeException { - public static final Key ErrNumberKey = Key.of( "ErrNumber" ); - /** * Internal expression error number. */ - public String errNumber = null; - public Position position = null; - public String sourceText = null; + protected String errNumber = null; + protected Position position = null; + protected String sourceText = null; /** * Constructor @@ -76,7 +74,7 @@ public Position getPosition() { public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( ErrNumberKey, errNumber ); + result.put( Key.errNumber, errNumber ); return result; } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/LockException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/LockException.java index 026e87ca5..b96b6f7ff 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/LockException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/LockException.java @@ -25,17 +25,14 @@ */ public class LockException extends BoxLangException { - public static final Key LockNameKey = Key.of( "LockName" ); - public static final Key LockOperationKey = Key.of( "LockOperation" ); - /** * Name of affected lock (if the lock is unnamed, the value is "anonymous"). */ - public String lockName = ""; + protected String lockName = ""; /** * Operation that failed (Timeout, Create Mutex, or Unknown). */ - public String lockOperation = ""; + protected String lockOperation = ""; /** * Constructor @@ -85,8 +82,8 @@ public String getLockOperation() { public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( LockNameKey, lockName ); - result.put( LockOperationKey, lockOperation ); + result.put( Key.lockName, lockName ); + result.put( Key.lockOperation, lockOperation ); return result; } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/MissingIncludeException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/MissingIncludeException.java index 4bb71ff7b..a1e9a7fe1 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/MissingIncludeException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/MissingIncludeException.java @@ -25,12 +25,10 @@ */ public class MissingIncludeException extends BoxLangException { - public static final Key MissingFileNameKey = Key.of( "missingFileName" ); - /** * Name of file that could not be included. */ - public String missingFileName = ""; + protected String missingFileName = ""; /** * Constructor @@ -66,7 +64,7 @@ public String getMissingFileName() { public IStruct dataAsStruct() { IStruct result = super.dataAsStruct(); - result.put( MissingFileNameKey, missingFileName ); + result.put( Key.missingFileName, missingFileName ); return result; } diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/ParseException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/ParseException.java index fbb72fb1b..fc26f4478 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/ParseException.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/ParseException.java @@ -28,7 +28,7 @@ */ public class ParseException extends BoxRuntimeException { - List issues; + protected List issues; /** * Constructor diff --git a/src/main/resources/dump/html/Null.bxm b/src/main/resources/dump/html/Null.bxm index 2fdd425c9..891b90c83 100644 --- a/src/main/resources/dump/html/Null.bxm +++ b/src/main/resources/dump/html/Null.bxm @@ -3,8 +3,7 @@

    - #label# - - NULL: + #label#: <null> diff --git a/src/main/resources/dump/html/Throwable.bxm b/src/main/resources/dump/html/Throwable.bxm index 0a0b4ee0b..a77bf72e1 100644 --- a/src/main/resources/dump/html/Throwable.bxm +++ b/src/main/resources/dump/html/Throwable.bxm @@ -2,39 +2,25 @@ expandRoot = expand ?: true; + -

    • - #encodeForHTML( method.getName() )# + #encodeForHTML( variables.method.getName() )# - 
    #encodeForHTML( method.toString() )#
    + 
    #encodeForHTML( variables.method.toString() )#
    - - - - - + - + class="d-none" > @@ -50,134 +36,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - + + + + + + + + +
    open aria-expanded="true"aria-expanded="false" - data-bx-toggle="siblings" - > - - #label# - - Error: #encodeForHTML( var.getType() )# - - open aria-expanded="true"aria-expanded="false" - data-bx-toggle="siblings" - > - - #label# - - Error: #var.getClass().getName()# - - open aria-expanded="true"aria-expanded="false" + data-bx-toggle="siblings" + > + + #label# - + Error: #encodeForHTML( exStruct.type )# + +
    - #encodeForHTML( var.getMessage() )# + #encodeForHTML( exStruct.message )#
    - Detail - -
    - #encodeForHTML( var.getDetail() ?: "n/a" )# -
    -
    - Tag Context - -
    - -
    -
    - SQL - -
    - -
    -
    - Query Error - -
    - -
    -
    - Where - -
    - -
    -
    - SQL State - -
    - -
    -
    - Native Error Code - -
    - -
    -
    - Cause + #encodeForHTML( UCFirst( thisKey ) )#
    - +
    - 
    #encodeForHTML( ExceptionUtil.getStackTraceAsString( var ) )#
    + 
    #encodeForHTML( exStruct.stackTrace )#
    + Cause + +
    + +
    +
    diff --git a/src/test/java/TestCases/components/BoxTemplateTest.java b/src/test/java/TestCases/components/BoxTemplateTest.java index f479514d6..4a712fdab 100644 --- a/src/test/java/TestCases/components/BoxTemplateTest.java +++ b/src/test/java/TestCases/components/BoxTemplateTest.java @@ -569,10 +569,10 @@ public void testThrowEverythingBagel() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @@ -593,10 +593,10 @@ public void testThrowAttributeCollection() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @Test @@ -617,8 +617,8 @@ public void testThrowingAnObjectViaAttributecollection() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.type ).isEqualTo( "custom" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getType() ).isEqualTo( "custom" ); } @Test diff --git a/src/test/java/TestCases/components/CFTemplateTest.java b/src/test/java/TestCases/components/CFTemplateTest.java index cdc648a36..aaa33a962 100644 --- a/src/test/java/TestCases/components/CFTemplateTest.java +++ b/src/test/java/TestCases/components/CFTemplateTest.java @@ -552,10 +552,10 @@ public void testThrowEverythingBagel() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @@ -569,10 +569,10 @@ public void testThrowEverythingBagelACFScript() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @Test @@ -592,10 +592,10 @@ public void testThrowAttributeCollection() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @Test @@ -615,10 +615,10 @@ public void testThrowAttributeCollectionACFScript() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.errorCode ).isEqualTo( "42" ); - assertThat( ce.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ce.type ).isEqualTo( "my.type" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getErrorCode() ).isEqualTo( "42" ); + assertThat( ce.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ce.getType() ).isEqualTo( "my.type" ); } @Test @@ -639,8 +639,8 @@ public void testThrowingAnObjectViaAttributecollection() { assertThat( ce.getMessage() ).isEqualTo( "my message" ); assertThat( ce.getCause() ).isNull(); - assertThat( ce.detail ).isEqualTo( "my detail" ); - assertThat( ce.type ).isEqualTo( "custom" ); + assertThat( ce.getDetail() ).isEqualTo( "my detail" ); + assertThat( ce.getType() ).isEqualTo( "custom" ); } @Test diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java index f4f3ecc94..31c8b80bd 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java @@ -123,11 +123,11 @@ public void testThrowEverything() { assertThat( e.getMessage() ).isEqualTo( "boom message" ); assertThat( e.getCause() ).isNotNull(); assertThat( e.getCause() ).isInstanceOf( java.lang.Exception.class ); - assertThat( e.detail ).isEqualTo( "boom detail" ); - assertThat( e.errorCode ).isEqualTo( "boom code" ); - assertThat( e.type ).isEqualTo( "boom.type" ); - assertThat( e.extendedInfo ).isInstanceOf( Array.class ); - assertThat( ( ( Array ) e.extendedInfo ).toArray( new String[ 0 ] ) ).isEqualTo( new String[] { "boom", "extended", "info" } ); + assertThat( e.getDetail() ).isEqualTo( "boom detail" ); + assertThat( e.getErrorCode() ).isEqualTo( "boom code" ); + assertThat( e.getType() ).isEqualTo( "boom.type" ); + assertThat( e.getExtendedInfo() ).isInstanceOf( Array.class ); + assertThat( ( ( Array ) e.getExtendedInfo() ).toArray( new String[ 0 ] ) ).isEqualTo( new String[] { "boom", "extended", "info" } ); } @Test From 19f54a3cb22f24a462de3b594c4a27449ff0e4dd Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 30 Dec 2024 12:56:03 -0600 Subject: [PATCH 080/161] BL-889 --- .../boxlang/runtime/types/exceptions/ExceptionUtil.java | 9 +++++++-- src/test/java/external/specs/QoQListsTest.cfc | 5 +---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java index 67a5f52e6..49cb7898b 100644 --- a/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/ExceptionUtil.java @@ -464,8 +464,13 @@ public static IStruct throwableToStruct( Throwable target ) { result.put( Key.type, target.getClass().getName() ); } - if ( result.containsKey( Key.suppressed ) && result.get( Key.suppressed ) instanceof Array sa && sa.isEmpty() ) { - result.remove( Key.suppressed ); + if ( result.containsKey( Key.suppressed ) ) { + Object oSuppressed = result.get( Key.suppressed ); + if ( oSuppressed != null && oSuppressed.getClass().isArray() ) { + if ( ( ( Object[] ) oSuppressed ).length == 0 ) { + result.remove( Key.suppressed ); + } + } } if ( result.containsKey( Key.cause ) && result.get( Key.cause ) == null ) { result.remove( Key.cause ); diff --git a/src/test/java/external/specs/QoQListsTest.cfc b/src/test/java/external/specs/QoQListsTest.cfc index 27a6e5bdb..a79e72aaa 100644 --- a/src/test/java/external/specs/QoQListsTest.cfc +++ b/src/test/java/external/specs/QoQListsTest.cfc @@ -9,10 +9,7 @@ component extends="testbox.system.BaseSpec"{ variables.interestingStringsAsAList = "a,c,e"; variables.interestingStringsAsAQuotedList = "'a','c','e'"; - variables.queryWithDataIn = Query( - id: [ 1 , 2 , 3 , 4 , 5 ], - value: [ 'a' , 'b' , 'c' , 'd' , 'e' ] - ); + variables.queryWithDataIn = QueryNew('id,value', 'integer,varchar',[[1,'a'],[2,'b'],[3,'c'],[4,'d'],[5,'e']]); } function run( testResults , testBox ) { From 110d07e08f335341ea2e4a34de1f180f125bebe9 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 30 Dec 2024 13:55:28 -0600 Subject: [PATCH 081/161] BL-894 --- .../boxlang/runtime/components/system/Loop.java | 12 +++++++++++- .../runtime/components/system/LoopTest.java | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java index 2d62f7b53..5cf48852f 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java @@ -20,17 +20,20 @@ import java.util.Collection; import java.util.Set; +import java.util.function.Supplier; import ortus.boxlang.runtime.components.Attribute; import ortus.boxlang.runtime.components.BoxComponent; import ortus.boxlang.runtime.components.Component; import ortus.boxlang.runtime.components.util.LoopUtil; +import ortus.boxlang.runtime.context.FunctionBoxContext; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.ExpressionInterpreter; import ortus.boxlang.runtime.dynamic.casters.BooleanCaster; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Array; +import ortus.boxlang.runtime.types.Closure; import ortus.boxlang.runtime.types.Function; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; @@ -170,7 +173,14 @@ private BodyResult _invokeTimes( IBoxContext context, Integer times, String item private BodyResult _invokeCondition( IBoxContext context, Function condition, ComponentBody body, IStruct executionState, String label ) { // Loop over array, executing body every time - while ( BooleanCaster.cast( context.invokeFunction( condition ) ) ) { + Supplier cond = () -> BooleanCaster.cast( context.invokeFunction( condition ) ); + // If our loop is inside a function, we need to use the original context to execute the condition, otherwise + // arguments and local scope lookups will be incorrect + IBoxContext declaringContext = ( ( Closure ) condition ).getDeclaringContext(); + if ( declaringContext instanceof FunctionBoxContext fbc ) { + cond = () -> BooleanCaster.cast( condition._invoke( fbc ) ); + } + while ( cond.get() ) { // Run the code inside of the output loop BodyResult bodyResult = processBody( context, body ); // IF there was a return statement inside our body, we early exit now diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java index d0b201814..0124e54aa 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java @@ -387,4 +387,21 @@ context, BoxSourceType.CFTEMPLATE ); } + @Test + public void testLoopCondition() { + instance.executeSource( + """ + function foo( required string name ) { + loop condition=arguments.name == "brad" { + return getFunctionCalledName(); + break; + } + } + + result = foo( "brad" ); + """, + context, BoxSourceType.BOXSCRIPT ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "foo" ); + } + } From 2e56c4024f3f366726c51261c09b88cd25aafebc Mon Sep 17 00:00:00 2001 From: Jacob Beers Date: Mon, 30 Dec 2024 23:27:13 -0600 Subject: [PATCH 082/161] BL-872 fix function return issue in ASM --- .../transformer/expression/BoxReturnTransformer.java | 7 ++++++- src/test/java/TestCases/phase3/ClassTest.java | 10 ++++++++++ src/test/java/TestCases/phase3/ClassWithInclude.cfc | 6 ++++++ src/test/java/TestCases/phase3/Mixin.cfm | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 src/test/java/TestCases/phase3/ClassWithInclude.cfc create mode 100644 src/test/java/TestCases/phase3/Mixin.cfm diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxReturnTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxReturnTransformer.java index 42026846b..77db62c81 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxReturnTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxReturnTransformer.java @@ -30,7 +30,9 @@ import ortus.boxlang.compiler.asmboxpiler.transformer.ReturnValueContext; import ortus.boxlang.compiler.asmboxpiler.transformer.TransformerContext; import ortus.boxlang.compiler.ast.BoxNode; +import ortus.boxlang.compiler.ast.statement.BoxFunctionDeclaration; import ortus.boxlang.compiler.ast.statement.BoxReturn; +import ortus.boxlang.compiler.ast.statement.component.BoxComponent; import ortus.boxlang.runtime.components.Component; public class BoxReturnTransformer extends AbstractTransformer { @@ -53,9 +55,12 @@ public List transform( BoxNode node, TransformerContext contex return AsmHelper.addLineNumberLabels( nodes, node ); } + BoxNode firstFound = node.getFirstNodeOfTypes( BoxFunctionDeclaration.class, BoxComponent.class ); + boolean preferFunction = firstFound instanceof BoxFunctionDeclaration; + if ( boxReturn.getExpression() == null ) { nodes.add( new InsnNode( Opcodes.ACONST_NULL ) ); - } else if ( transpiler.isInsideComponent() ) { + } else if ( transpiler.isInsideComponent() && !preferFunction ) { nodes.addAll( transpiler.transform( boxReturn.getExpression(), TransformerContext.NONE, ReturnValueContext.VALUE_OR_NULL ) ); nodes.add( new MethodInsnNode( diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index 4bad4b160..eb725a9ea 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1600,6 +1600,16 @@ public void testClassLocatorInStaticInitializer() { context ); } + @DisplayName( "Class with include" ) + @Test + public void testClassWithInclude() { + instance.executeSource( + """ + new src.test.java.TestCases.phase3.ClassWithInclude() + """, + context ); + } + @DisplayName( "properties not inherited in metadata" ) @Test public void testPropertiesNotInheritedInMetadata() { diff --git a/src/test/java/TestCases/phase3/ClassWithInclude.cfc b/src/test/java/TestCases/phase3/ClassWithInclude.cfc new file mode 100644 index 000000000..9f04f1794 --- /dev/null +++ b/src/test/java/TestCases/phase3/ClassWithInclude.cfc @@ -0,0 +1,6 @@ +component { + include template="Mixin.cfm"; + + foo() + +} \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/Mixin.cfm b/src/test/java/TestCases/phase3/Mixin.cfm new file mode 100644 index 000000000..90536310e --- /dev/null +++ b/src/test/java/TestCases/phase3/Mixin.cfm @@ -0,0 +1,7 @@ + + + struct function foo() { + return {} + } + + \ No newline at end of file From 6728ae410adcf2a652df9318031cc5fb80c36baa Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 1 Jan 2025 12:56:24 -0600 Subject: [PATCH 083/161] BL-898 --- .../boxlang/runtime/jdbc/ExecutedQuery.java | 107 ++-- .../boxlang/runtime/jdbc/PendingQuery.java | 541 +++++++++++++----- .../boxlang/runtime/jdbc/QueryParameter.java | 8 +- .../runtime/jdbc/qoq/QoQExecutionService.java | 2 +- src/test/java/external/specs/QoQListsTest.cfc | 48 +- .../ortus/boxlang/compiler/QoQParseTest.java | 43 +- .../bifs/global/jdbc/BaseJDBCTest.java | 3 +- .../bifs/global/jdbc/QueryExecuteTest.java | 54 +- .../runtime/components/jdbc/QueryTest.java | 2 +- .../boxlang/runtime/jdbc/DataSourceTest.java | 2 +- 10 files changed, 519 insertions(+), 291 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java index a5a60aa1a..3258a64b8 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ExecutedQuery.java @@ -94,63 +94,79 @@ public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery Object generatedKey = null; Query results = null; int recordCount = 0; + int affectedCount = -1; if ( statement instanceof QoQStatement qs ) { results = qs.getQueryResult(); } else { - try ( ResultSet rs = statement.getResultSet() ) { - results = Query.fromResultSet( rs ); - } catch ( SQLException e ) { - throw new DatabaseException( e.getMessage(), e ); - } - } - if ( hasResults ) { - recordCount = results.size(); - } + // Loop over results until we find a result set, or run out of results + while ( true ) { + if ( hasResults ) { + try ( ResultSet rs = statement.getResultSet() ) { + results = Query.fromResultSet( rs ); + recordCount = results.size(); + break; + } catch ( SQLException e ) { + throw new DatabaseException( e.getMessage(), e ); + } + } else { - // Capture generated keys, if any. - try { - if ( !hasResults ) { - try { - int affectedCount = statement.getUpdateCount(); - if ( affectedCount > -1 ) { - recordCount = affectedCount; + // Capture generated keys, if any. + try { + try { + affectedCount = statement.getUpdateCount(); + if ( affectedCount > -1 ) { + recordCount = affectedCount; + try ( ResultSet keys = statement.getGeneratedKeys() ) { + if ( keys != null && keys.next() ) { + generatedKey = keys.getObject( 1 ); + } + } catch ( SQLException e ) { + // @TODO: drop the message check, since it doesn't support alternate languages. + if ( e.getMessage().contains( "The statement must be executed before any results can be obtained." ) ) { + logger.info( + "SQL Server threw an error when attempting to retrieve generated keys. Am ignoring the error - no action is required. Error : [{}]", + e.getMessage() ); + } else { + logger.warn( "Error getting generated keys", e ); + } + } + } + } catch ( SQLException t ) { + logger.error( "Error getting update count", t ); + } + } catch ( NullPointerException e ) { + // This is likely due to Hikari wrapping a null ResultSet. + // There should not be a null ResultSet returned from getGeneratedKeys + // (https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#getGeneratedKeys--) + // but some JDBC drivers do anyway. + // Since Hikari wraps the null value, we can't get access to it, + // so instead we have to catch it here and ignore it. + // We do check the message to try to be very particular about what NullPointerExceptions we are catching + if ( !e.getMessage().equals( "Cannot invoke \"java.sql.ResultSet.next()\" because \"this.delegate\" is null" ) ) { + throw e; + } } - } catch ( SQLException t ) { - logger.error( "Error getting update count", t ); } - try ( ResultSet keys = statement.getGeneratedKeys() ) { - if ( keys != null && keys.next() ) { - generatedKey = keys.getObject( 1 ); - } + // If we have no results and no affected count, we're done. + if ( !hasResults && affectedCount == -1 ) { + break; + } + + // Otherwise, look for another result set or update count. + try { + hasResults = statement.getMoreResults(); } catch ( SQLException e ) { - // @TODO: drop the message check, since it doesn't support alternate languages. - if ( e.getMessage().contains( "The statement must be executed before any results can be obtained." ) ) { - logger.info( - "SQL Server threw an error when attempting to retrieve generated keys. Am ignoring the error - no action is required. Error : [{}]", - e.getMessage() ); - } else { - logger.warn( "Error getting generated keys", e ); - } + break; } - } - } catch ( NullPointerException e ) { - // This is likely due to Hikari wrapping a null ResultSet. - // There should not be a null ResultSet returned from getGeneratedKeys - // (https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#getGeneratedKeys--) - // but some JDBC drivers do anyway. - // Since Hikari wraps the null value, we can't get access to it, - // so instead we have to catch it here and ignore it. - // We do check the message to try to be very particular about what NullPointerExceptions we are catching - if ( !e.getMessage().equals( "Cannot invoke \"java.sql.ResultSet.next()\" because \"this.delegate\" is null" ) ) { - throw e; - } + + } // /while } IStruct queryMeta = Struct.of( "cached", false, "cacheKey", pendingQuery.getCacheKey(), - "sql", pendingQuery.getOriginalSql(), + "sql", pendingQuery.getSQLWithParamValues(), "sqlParameters", Array.fromList( pendingQuery.getParameterValues() ), "executionTime", executionTime, "recordCount", recordCount @@ -160,6 +176,11 @@ public static ExecutedQuery fromPendingQuery( @Nonnull PendingQuery pendingQuery queryMeta.put( "generatedKey", generatedKey ); } + // If we only had an update or insert, we need an empty query object to return + if ( results == null ) { + results = new Query(); + } + // important that we set the metadata on the Query object for later getBoxMeta(), i.e. $bx.meta calls. results.setMetadata( queryMeta ); ExecutedQuery executedQuery = new ExecutedQuery( results, generatedKey ); diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java index 9839b4e4c..6e1ba0e02 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java @@ -19,8 +19,9 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; -import java.util.regex.Matcher; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -30,11 +31,11 @@ import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.cache.providers.ICacheProvider; -import ortus.boxlang.runtime.components.jdbc.QueryParam; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.Attempt; import ortus.boxlang.runtime.dynamic.casters.ArrayCaster; import ortus.boxlang.runtime.dynamic.casters.CastAttempt; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.dynamic.casters.StructCaster; import ortus.boxlang.runtime.events.BoxEvent; import ortus.boxlang.runtime.scopes.Key; @@ -44,14 +45,14 @@ import ortus.boxlang.runtime.types.Query; import ortus.boxlang.runtime.types.QueryColumnType; import ortus.boxlang.runtime.types.Struct; -import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.types.exceptions.DatabaseException; import ortus.boxlang.runtime.types.util.ListUtil; -import ortus.boxlang.runtime.util.RegexBuilder; /** - * This class represents a query and any parameters/bindings before being executed. - * After calling {@link #execute(ConnectionManager,IBoxContext)}, it returns an {@link ExecutedQuery} with a reference to this object. + * This class represents a query and any parameters/bindings before being + * executed. + * After calling {@link #execute(ConnectionManager,IBoxContext)}, it returns an + * {@link ExecutedQuery} with a reference to this object. */ public class PendingQuery { @@ -76,21 +77,33 @@ public class PendingQuery { /** * The SQL string to execute. *

    - * If this SQL has parameters, they should be represented either as question marks (`?`) + * If this SQL has parameters, they should be represented either as question + * marks (`?`) * or as named bindings, prefixed with a colon (`:`) */ private @Nonnull String sql; + private List SQLWithParamTokens = new ArrayList<>(); /** - * The original SQL provided to the constructor. When constructing a `PendingQuery` with an - * {@link IStruct} instance, the named parameters — prefixed with a colon (`:`) in the SQL string — - * are replaced with question marks to work with the JDBC query. The SQL string with question marks - * is set as `sql` and the original SQL provided with named parameters is set as `originalSql`. + * SQL string broken up into segments so we can build a fully + */ + private @Nonnull String SQLWithParamValues; + + /** + * The original SQL provided to the constructor. When constructing a + * `PendingQuery` with an + * {@link IStruct} instance, the named parameters — prefixed with a colon (`:`) + * in the SQL string — + * are replaced with question marks to work with the JDBC query. The SQL string + * with question marks + * is set as `sql` and the original SQL provided with named parameters is set as + * `originalSql`. */ private @Nonnull final String originalSql; /** - * A List of QueryParameter instances. The store the value and SQL type of the parameters, in order. + * A List of QueryParameter instances. The store the value and SQL type of the + * parameters, in order. * * @see QueryParameter */ @@ -102,7 +115,8 @@ public class PendingQuery { private QueryOptions queryOptions; /** - * The cache key for this query, determined from the combined hash of the SQL string and parameter values. + * The cache key for this query, determined from the combined hash of the SQL + * string and parameter values. */ private String cacheKey; @@ -118,47 +132,51 @@ public class PendingQuery { */ /** - * Creates a new PendingQuery instance from a SQL string, a list of parameters, and the original SQL string. + * Creates a new PendingQuery instance from a SQL string, a list of parameters, + * and the original SQL string. * * @param sql The SQL string to execute - * @param bindings An array or struct of {@link QueryParameter} to use as bindings. + * @param bindings An array or struct of {@link QueryParameter} to use as + * bindings. * @param queryOptions QueryOptions object denoting the options for this query. */ public PendingQuery( @Nonnull String sql, Object bindings, QueryOptions queryOptions ) { logger.debug( "Building new PendingQuery from SQL: [{}] and options: [{}]", sql, queryOptions.toStruct() ); /** - * `onQueryBuild()` interception: Use this to modify query parameters or options before the query is executed. + * `onQueryBuild()` interception: Use this to modify query parameters or options + * before the query is executed. * * The event args will contain the following keys: * * - sql : The original SQL string * - parameters : The parameters to be used in the query * - pendingQuery : The BoxLang query class used to build and execute queries - * - options : The QueryOptions class populated with query options from `queryExecute()` or `` + * - options : The QueryOptions class populated with query options from + * `queryExecute()` or `` */ IStruct eventArgs = Struct.of( "sql", sql.trim(), "bindings", bindings, "pendingQuery", this, - "options", queryOptions - ); + "options", queryOptions ); interceptorService.announce( BoxEvent.ON_QUERY_BUILD, eventArgs ); // We set instance data from the event args so interceptors can modify them. - this.sql = eventArgs.getAsString( Key.sql ); - this.originalSql = eventArgs.getAsString( Key.sql ); - this.parameters = processBindings( eventArgs.get( Key.of( "bindings" ) ) ); - this.sql = massageSQL(); - this.queryOptions = eventArgs.getAs( QueryOptions.class, Key.options ); + this.sql = eventArgs.getAsString( Key.sql ); + this.SQLWithParamValues = this.sql; + this.originalSql = eventArgs.getAsString( Key.sql ); + this.parameters = processBindings( eventArgs.get( Key.of( "bindings" ) ) ); + this.queryOptions = eventArgs.getAs( QueryOptions.class, Key.options ); // Create a cache key with a default or via the passed options. - this.cacheKey = getOrComputeCacheKey(); - this.cacheProvider = BoxRuntime.getInstance().getCacheService().getCache( this.queryOptions.cacheProvider ); + this.cacheKey = getOrComputeCacheKey(); + this.cacheProvider = BoxRuntime.getInstance().getCacheService().getCache( this.queryOptions.cacheProvider ); } /** - * Creates a new PendingQuery instance from a SQL string and a list of parameters. + * Creates a new PendingQuery instance from a SQL string and a list of + * parameters. * This constructor uses the provided SQL string as the original SQL. * * @param sql The SQL string to execute @@ -176,7 +194,10 @@ public PendingQuery( @Nonnull String sql, @Nonnull List paramete /** * Returns the cache key for this query. *

    - * If a custom cache key was provided in the query options, it will be used. Otherwise, a cache key will be generated from a combined hash of the SQL string,parameter values, and relevant query options such as the `datasource`, `username`, and + * If a custom cache key was provided in the query options, it will be used. + * Otherwise, a cache key will be generated from a combined hash of the SQL + * string,parameter values, and relevant query options such as the `datasource`, + * `username`, and * `password`. */ private String getOrComputeCacheKey() { @@ -204,10 +225,13 @@ public String getCacheKey() { } /** - * Processes the bindings provided to the constructor and returns a list of {@link QueryParameter} instances. - * Will also modify the SQL string to replace named parameters with positional placeholders. + * Processes the bindings provided to the constructor and returns a list of + * {@link QueryParameter} instances. + * Will also modify the SQL string to replace named parameters with positional + * placeholders. * - * @param bindings The bindings to process. This can be an {@link Array} of values or a {@link IStruct} of named parameters. + * @param bindings The bindings to process. This can be an {@link Array} of + * values or a {@link IStruct} of named parameters. * * @return A list of {@link QueryParameter} instances. */ @@ -217,82 +241,236 @@ private List processBindings( Object bindings ) { } CastAttempt castAsArray = ArrayCaster.attempt( bindings ); if ( castAsArray.wasSuccessful() ) { - return buildParameterList( castAsArray.getOrFail() ); + Array castedArray = castAsArray.get(); + // Short circuit for empty arrays to simplify our checks below + if ( castedArray.isEmpty() ) { + return new ArrayList<>(); + } + // An array could be an array of structs where a name is specified in each struct, which we massage back into a struct + int foundNames = 0; + IStruct possibleStruct = Struct.of(); + for ( Object item : castedArray ) { + CastAttempt itemStructAttempt = StructCaster.attempt( item ); + if ( itemStructAttempt.wasSuccessful() ) { + IStruct itemStruct = itemStructAttempt.get(); + if ( itemStruct.containsKey( Key._NAME ) ) { + foundNames++; + possibleStruct.put( Key.of( itemStruct.get( Key._NAME ) ), itemStruct ); + } + } + } + // All items in the array are structs with names + if ( foundNames == castedArray.size() ) { + return buildParameterList( null, possibleStruct ); + } else if ( foundNames > 0 ) { + throw new DatabaseException( "Invalid query params passed as array of structs. Some structs have a name, some do not." ); + } + // No structs with names were found, or possbly no structs were found at all! + return buildParameterList( castedArray, null ); } CastAttempt castAsStruct = StructCaster.attempt( bindings ); if ( castAsStruct.wasSuccessful() ) { - return buildParameterList( castAsStruct.getOrFail() ); + return buildParameterList( null, castAsStruct.get() ); } // We always have bindings, since we exit early if there are none String className = bindings.getClass().getName(); - throw new BoxRuntimeException( "Invalid type for params. Expected array or struct. Received: " + className ); + throw new DatabaseException( "Invalid type for query params. Expected array or struct. Received: " + className ); } /** - * Process an array of query bindings into a list of {@link QueryParameter} instances. - * - * @param parameters An {@link Array} of `queryparam` {@link IStruct} instances to convert to {@link QueryParameter} instances and use as bindings. - */ - private List buildParameterList( @Nonnull Array parameters ) { - return parameters.stream().map( QueryParameter::fromAny ).collect( Collectors.toList() ); - } - - /** - * Process a struct of named query bindings into a list of {@link QueryParameter} instances. + * Process a struct of named query bindings into a list of + * {@link QueryParameter} instances. *

    - * Also performs SQL string replacement to convert named parameters to positional placeholders. + * Also performs SQL string replacement to convert named parameters to + * positional placeholders. * * @param sql The SQL string to execute - * @param parameters An `IStruct` of `String` `name` to either an `Object` `value` or a `queryparam` `IStruct`. + * @param parameters An `IStruct` of `String` `name` to either an `Object` + * `value` or a `queryparam` `IStruct`. */ - private List buildParameterList( @Nonnull IStruct parameters ) { - List params = new ArrayList<>(); - Matcher matcher = RegexBuilder.SQL_PARAMETER.matcher( sql ); - while ( matcher.find() ) { - String paramName = matcher.group(); - paramName = paramName.substring( 1 ); - Object paramValue = parameters.get( paramName ); - if ( paramValue == null ) { - throw new DatabaseException( "Missing param in query: [" + paramName + "]. SQL: " + sql ); + @SuppressWarnings( { "null", "unchecked" } ) + private List buildParameterList( Array positionalParameters, IStruct namedParameters ) { + List params = new ArrayList<>(); + // Short circuit for no parameters + if ( positionalParameters == null && namedParameters == null ) { + return params; + } else if ( positionalParameters != null && positionalParameters.isEmpty() ) { + return params; + } else if ( namedParameters != null && namedParameters.isEmpty() ) { + return params; + } + + boolean isPositional = positionalParameters != null; + String SQL = this.sql; + // This is the SQL string with the named parameters replaced with positional placeholders + StringBuilder newSQL = new StringBuilder(); + // This is the name of the current named parameter being processed + StringBuilder paramName = new StringBuilder(); + // This is the current SQL token being processed. We'll save these for later when we apply the parameters. + // We could techincally finalize this string now, but we'd end up casting all the values twice which seems inefficient. + StringBuilder SQLWithParamToken = new StringBuilder(); + // Track the named params we encounter for validation below + Set foundNamedParams = new HashSet<>(); + + // 0 = Default state, processing SQL + // 1 = Inside a string literal + // 2 = Inside a single line comment + // 3 = Inside a multi-line comment + // 4 = Inside a named parameter + int state = 0; + + // This should always match params.size(), but is a little easier to use + int paramsEncountered = 0; + // Pop this into a lambda so we can re-use it for the last named parameter + Runnable processNamed = () -> { + SQLWithParamTokens.add( SQLWithParamToken.toString() ); + SQLWithParamToken.setLength( 0 ); + Key finalParamName = Key.of( paramName.toString() ); + if ( isPositional ) { + throw new DatabaseException( + "Named parameter [:" + finalParamName.getName() + "] found in query with positional parameters." ); + } else { + if ( namedParameters.containsKey( finalParamName ) ) { + QueryParameter newParam = QueryParameter.fromAny( namedParameters.get( finalParamName ) ); + foundNamedParams.add( finalParamName ); + params.add( newParam ); + // List params add ?, ?, ? etc. to the SQL string + if ( newParam.isListParam() ) { + List values = ( List ) newParam.getValue(); + newSQL.append( values.stream().map( v -> "?" ).collect( Collectors.joining( ", " ) ) ); + } else { + newSQL.append( "?" ); + } + } else { + throw new DatabaseException( + "Named parameter [:" + finalParamName.getName() + "] not provided to query." ); + } + } + }; + + for ( int i = 0; i < SQL.length(); i++ ) { + char c = SQL.charAt( i ); + + switch ( state ) { + // Default state, processing SQL + case 0 : { + if ( c == '\'' ) { + // If we've reached a ' then we're inside a string literal + state = 1; + } else if ( c == '-' && i < SQL.length() - 1 && SQL.charAt( i + 1 ) == '-' ) { + // If we've reached a -- then we're inside a single line comment + state = 2; + } else if ( c == '/' && i < SQL.length() - 1 && SQL.charAt( i + 1 ) == '*' ) { + // If we've reached a /* then we're inside a multi-line comment + state = 3; + } else if ( c == '?' ) { + // We've encountered a positional parameter + paramsEncountered++; + if ( isPositional ) { + if ( paramsEncountered > positionalParameters.size() ) { + throw new DatabaseException( "Too few positional parameters [" + positionalParameters.size() + + "] provided for query having at least [" + paramsEncountered + "] '?' char(s)." ); + } + + SQLWithParamTokens.add( SQLWithParamToken.toString() ); + SQLWithParamToken.setLength( 0 ); + var newParam = QueryParameter.fromAny( positionalParameters.get( paramsEncountered - 1 ) ); + List values; + // List params add ?, ?, ? etc. to the SQL string + if ( newParam.isListParam() && ( values = ( List ) newParam.getValue() ).size() > 1 ) { + newSQL.append( "?, ".repeat( values.size() - 1 ) ); + } + params.add( newParam ); + // append here and break so the ? doesn't go into the SQLWithParamToken + newSQL.append( c ); + break; + } else { + throw new DatabaseException( "Positional parameter [?] found in query with named parameters." ); + } + } else if ( c == ':' ) { + // We've encountered a named parameter + state = 4; + // Do not append anything + break; + } + newSQL.append( c ); + SQLWithParamToken.append( c ); + break; + } + // Inside a string literal + case 1 : { + // If we've reached the ending ' and it wasn't escaped as \' then we're done + if ( c == '\'' && ( i == SQL.length() - 1 || SQL.charAt( i + 1 ) != '\'' ) ) { + state = 0; + // if we reached ' but the next char is also ' then this is just an escaped '' + // Append them both and move on + } else if ( c == '\'' && i < SQL.length() - 1 && SQL.charAt( i + 1 ) == '\'' ) { + newSQL.append( c ); // Append the first single quote + SQLWithParamToken.append( c ); + c = SQL.charAt( ++i ); // Skip the next single quote + } + newSQL.append( c ); + SQLWithParamToken.append( c ); + break; + } + // Inside a single line comment + case 2 : { + if ( c == '\n' || c == '\r' ) { + state = 0; + } + newSQL.append( c ); + SQLWithParamToken.append( c ); + break; + } + // Inside a multi-line comment + case 3 : { + if ( c == '*' && i < SQL.length() - 1 && SQL.charAt( i + 1 ) == '/' ) { + state = 0; + newSQL.append( c ); + SQLWithParamToken.append( c ); + c = SQL.charAt( ++i ); + } + newSQL.append( c ); + SQLWithParamToken.append( c ); + break; + } + // Inside a named parameter + case 4 : { + if ( ! ( Character.isLetterOrDigit( c ) || c == '_' ) ) { + processNamed.run(); + paramName.setLength( 0 ); + // reset the state and backup to re-precess the next char again + state = 0; + i--; + break; + } + paramName.append( c ); + break; + } } - params.add( QueryParameter.fromAny( paramName, paramValue ) ); } - return params; - } - /** - * Massage the SQL string, replacing named parameters (`:name`) with positional placeholders (`?`). - * - * Query params do not insert named params, they insert positional params, and they already take care of inserting the correct number of placeholder tokens. - * - * @see {@link QueryParam#_invoke} for queryparam placeholder token insertion - */ - private String massageSQL() { - if ( parameters.isEmpty() ) { - return sql; + // If named param is the last thing in the query + if ( state == 4 ) { + processNamed.run(); } - // Replace the params as we go, so we can process positionally in the order they appear in the SQL string. - String tempPlaceholder = "____QUESTION_MARK____"; - for ( QueryParameter p : parameters ) { - String sqlReplacement = tempPlaceholder; - if ( p.isListParam() ) { - Array v = ( Array ) p.getValue(); - sqlReplacement = v.stream().map( param -> tempPlaceholder ).collect( Collectors.joining( "," ) ); - } - String placeholder = "\\?"; - if ( p.getName() != null ) { - placeholder = ":" + p.getName(); - // If this is a named param, replace all instances of it - sql = RegexBuilder.of( sql, placeholder ).replaceAllAndGet( sqlReplacement ); - } else { - // If it's a positional param, replace the first instance - sql = RegexBuilder.of( sql, placeholder ).replaceFirstAndGet( sqlReplacement ); - } + + // Make sure positional params were all used + if ( isPositional && positionalParameters.size() > paramsEncountered ) { + throw new DatabaseException( "Too many positional parameters [" + positionalParameters.size() + + "] provided for query having only [" + paramsEncountered + "] '?' char(s)." ); + } else if ( !isPositional && namedParameters.keySet().size() != foundNamedParams.size() ) { + // Make sure all named params were used + Set missingParams = new HashSet<>( namedParameters.keySet() ); + missingParams.removeAll( foundNamedParams ); + throw new DatabaseException( "Named parameter(s) [" + missingParams + "] provided to query were not used." ); } - this.sql = RegexBuilder.of( sql, tempPlaceholder ).replaceAllAndGet( "?" ); - return sql; + + SQLWithParamTokens.add( SQLWithParamToken.toString() ); + this.sql = newSQL.toString(); + return params; } /** @@ -304,28 +482,51 @@ private String massageSQL() { return this.originalSql; } + /** + * Return final SQL with paramter values + * + * @return + */ + public @Nonnull String getSQLWithParamValues() { + return this.SQLWithParamValues; + } + /** * Returns a list of parameter `Object` values from the `List`. + * If a parameter is a list type, its values are expanded into the main list. * * @return A list of parameter values as `Object`s. */ public @Nonnull List getParameterValues() { - return this.parameters.stream().map( QueryParameter::getValue ).collect( Collectors.toList() ); + List values = new ArrayList<>(); + for ( QueryParameter param : this.parameters ) { + if ( param.isListParam() ) { + values.addAll( ( List ) param.getValue() ); + } else { + values.add( param.getValue() ); + } + } + return values; } /** - * Executes the PendingQuery using the provided ConnectionManager and returns the results in an {@link ExecutedQuery} instance. + * Executes the PendingQuery using the provided ConnectionManager and returns + * the results in an {@link ExecutedQuery} instance. * - * @param connectionManager The ConnectionManager instance to use for getting connections from the current context. + * @param connectionManager The ConnectionManager instance to use for getting + * connections from the current context. * - * @throws DatabaseException If a {@link SQLException} occurs, wraps it in a DatabaseException and throws. + * @throws DatabaseException If a {@link SQLException} occurs, wraps it in a + * DatabaseException and throws. * - * @return An ExecutedQuery instance with the results of this JDBC execution, as well as a link to this PendingQuery instance. + * @return An ExecutedQuery instance with the results of this JDBC execution, as + * well as a link to this PendingQuery instance. * * @see ExecutedQuery */ public @Nonnull ExecutedQuery execute( ConnectionManager connectionManager, IBoxContext context ) { - // We do an early cache check here to avoid the overhead of creating a connection if we already have a matching cached query. + // We do an early cache check here to avoid the overhead of creating a + // connection if we already have a matching cached query. if ( isCacheable() ) { logger.debug( "Checking cache for query: {}", this.cacheKey ); Attempt cachedQuery = cacheProvider.get( this.cacheKey ); @@ -346,26 +547,33 @@ private String massageSQL() { } /** - * Executes the PendingQuery on a given {@link Connection} and returns the results in an {@link ExecutedQuery} instance. + * Executes the PendingQuery on a given {@link Connection} and returns the + * results in an {@link ExecutedQuery} instance. * - * @param connection The Connection instance to use for executing the query. It is the responsibility of the caller to close the connection after this method returns. + * @param connection The Connection instance to use for executing the query. It + * is the responsibility of the caller to close the connection + * after this method returns. * - * @throws DatabaseException If a {@link SQLException} occurs, wraps it in a DatabaseException and throws. + * @throws DatabaseException If a {@link SQLException} occurs, wraps it in a + * DatabaseException and throws. * - * @return An ExecutedQuery instance with the results of this JDBC execution, as well as a link to this PendingQuery instance. + * @return An ExecutedQuery instance with the results of this JDBC execution, as + * well as a link to this PendingQuery instance. * * @see ExecutedQuery */ public @Nonnull ExecutedQuery execute( Connection connection, IBoxContext context ) { if ( isCacheable() ) { - // we use separate get() and set() calls over a .getOrSet() so we can run `.setIsCached()` on discovered/cached results. + // we use separate get() and set() calls over a .getOrSet() so we can run + // `.setIsCached()` on discovered/cached results. Attempt cachedQuery = this.cacheProvider.get( this.cacheKey ); if ( cachedQuery.isPresent() ) { return respondWithCachedQuery( ( ExecutedQuery ) cachedQuery.get() ); } ExecutedQuery executedQuery = executeStatement( connection, context ); - this.cacheProvider.set( this.cacheKey, executedQuery, this.queryOptions.cacheTimeout, this.queryOptions.cacheLastAccessTimeout ); + this.cacheProvider.set( this.cacheKey, executedQuery, this.queryOptions.cacheTimeout, + this.queryOptions.cacheLastAccessTimeout ); return executedQuery; } return executeStatement( connection, context ); @@ -374,45 +582,42 @@ private String massageSQL() { /** * Generate and execute a JDBC statement using the provided connection. *

    - * * If query parameters are present, a {@link PreparedStatement} will be utilized and populated with the paremeter bindings. Otherwise, a standard {@link Statement} object will be used. + * * If query parameters are present, a {@link PreparedStatement} will be + * utilized and populated with the paremeter bindings. Otherwise, a standard + * {@link Statement} object will be used. * * Will announce a `PRE_QUERY_EXECUTE` event before executing the query. */ private ExecutedQuery executeStatement( Connection connection, IBoxContext context ) { try { - ArrayList queries = new ArrayList<>(); - for ( String sqlStatement : this.sql.split( ";" ) ) { - try ( - Statement statement = this.parameters.isEmpty() - ? connection.createStatement() - : connection.prepareStatement( this.sql, Statement.RETURN_GENERATED_KEYS ); ) { - - applyParameters( statement, context ); - applyStatementOptions( statement ); - - interceptorService.announce( - BoxEvent.PRE_QUERY_EXECUTE, - Struct.of( - "sql", this.sql, - "bindings", getParameterValues(), - "pendingQuery", this - ) - ); - - long startTick = System.currentTimeMillis(); - boolean hasResults = statement instanceof PreparedStatement preparedStatement - ? preparedStatement.execute() - : statement.execute( sqlStatement, Statement.RETURN_GENERATED_KEYS ); - long endTick = System.currentTimeMillis(); - - queries.add( ExecutedQuery.fromPendingQuery( - this, - statement, - endTick - startTick, - hasResults - ) ); - } + + String sqlStatement = this.sql; + try ( + Statement statement = this.parameters.isEmpty() + ? connection.createStatement() + : connection.prepareStatement( sqlStatement, Statement.RETURN_GENERATED_KEYS ); ) { + + applyParameters( statement, context ); + applyStatementOptions( statement ); + + interceptorService.announce( + BoxEvent.PRE_QUERY_EXECUTE, + Struct.of( + "sql", sqlStatement, + "bindings", getParameterValues(), + "pendingQuery", this ) ); + + long startTick = System.currentTimeMillis(); + boolean hasResults = statement instanceof PreparedStatement preparedStatement + ? preparedStatement.execute() + : statement.execute( sqlStatement, Statement.RETURN_GENERATED_KEYS ); + long endTick = System.currentTimeMillis(); + + return ExecutedQuery.fromPendingQuery( + this, + statement, + endTick - startTick, + hasResults ); } - return queries.getFirst(); } catch ( SQLException e ) { String detail = ""; if ( e.getCause() != null ) { @@ -426,15 +631,17 @@ private ExecutedQuery executeStatement( Connection connection, IBoxContext conte originalSql, null, // queryError ListUtil.asString( Array.fromList( this.getParameterValues() ), "," ), // where - e - ); + e ); } } /** - * Helper method to respond with an ExecutedQuery instance from the given query cache lookup. + * Helper method to respond with an ExecutedQuery instance from the given query + * cache lookup. *

    - * This method assumes cachedQuery.isPresent() has already been checked, and populates the ExecutedQuery instance with the query cache metadata, such as cacheKey, cacheProvider, etc. + * This method assumes cachedQuery.isPresent() has already been checked, and + * populates the ExecutedQuery instance with the query cache metadata, such as + * cacheKey, cacheProvider, etc. */ private ExecutedQuery respondWithCachedQuery( ExecutedQuery cachedQuery ) { logger.debug( "Query is present, returning cached result: {}", this.cacheKey ); @@ -443,8 +650,7 @@ private ExecutedQuery respondWithCachedQuery( ExecutedQuery cachedQuery ) { "cacheKey", this.cacheKey, "cacheProvider", this.cacheProvider.getName().toString(), "cacheTimeout", this.queryOptions.cacheTimeout, - "cacheLastAccessTimeout", this.queryOptions.cacheLastAccessTimeout - ); + "cacheLastAccessTimeout", this.queryOptions.cacheLastAccessTimeout ); Query results = cachedQuery.getResults(); Struct queryMeta = new Struct( cachedQuery.getQueryMeta() ); queryMeta.addAll( cacheMeta ); @@ -453,9 +659,11 @@ private ExecutedQuery respondWithCachedQuery( ExecutedQuery cachedQuery ) { } /** - * Apply the parameter bindings to the provided {@link Statement} instance. - *

    - * Will only take action if 1) there are parameters to apply, and 2) the Statement object is a PreparedStatement. + * If this is a paramaterized query, apply the parameters to the provided statement. + * We will also take this opportunity to finalize the list of SQL tokens with the + * final param values to build the effective SQL string. + * + * @throws SQLException */ private void applyParameters( Statement statement, IBoxContext context ) throws SQLException { if ( this.parameters.isEmpty() ) { @@ -463,14 +671,22 @@ private void applyParameters( Statement statement, IBoxContext context ) throws } if ( statement instanceof PreparedStatement preparedStatement ) { + StringBuilder SQLWithParamValues = new StringBuilder(); // The param index starts from 1 - int parameterIndex = 1; + int parameterIndex = 1; + int SQLParamIndex = 0; for ( QueryParameter param : this.parameters ) { + SQLWithParamValues.append( SQLWithParamTokens.get( SQLParamIndex ) ); Integer scaleOrLength = param.getScaleOrLength(); if ( param.isListParam() ) { - Array list = ( Array ) param.getValue(); + var i = 1; + Array list = ( Array ) param.getValue(); for ( Object value : list ) { Object casted = QueryColumnType.toSQLType( param.getType(), value, context ); + emitValueToSQL( SQLWithParamValues, casted, param.getType() ); + if ( i < list.size() ) { + SQLWithParamValues.append( ", " ); + } if ( scaleOrLength == null ) { preparedStatement.setObject( parameterIndex, casted, param.getSqlTypeAsInt() ); } else { @@ -478,23 +694,51 @@ private void applyParameters( Statement statement, IBoxContext context ) throws scaleOrLength ); } parameterIndex++; + i++; } } else { + Object value = param.toSQLType( context ); + emitValueToSQL( SQLWithParamValues, value, param.getType() ); if ( scaleOrLength == null ) { - preparedStatement.setObject( parameterIndex, param.toSQLType( context ), param.getSqlTypeAsInt() ); + preparedStatement.setObject( parameterIndex, value, param.getSqlTypeAsInt() ); } else { - preparedStatement.setObject( parameterIndex, param.toSQLType( context ), param.getSqlTypeAsInt(), scaleOrLength ); + preparedStatement.setObject( parameterIndex, value, param.getSqlTypeAsInt(), + scaleOrLength ); } parameterIndex++; } + SQLParamIndex++; } + SQLWithParamValues.append( SQLWithParamTokens.get( SQLParamIndex ) ); + this.SQLWithParamValues = SQLWithParamValues.toString(); + SQLWithParamTokens.clear(); + } + } + + /** + * Emit a value to the SQL string, quoting it if necessary. + * + * @param SQLWithParamValues The SQL string to append the value to. + * @param value The value to append. + * @param type The type of the value. + */ + private void emitValueToSQL( StringBuilder SQLWithParamValues, Object value, QueryColumnType type ) { + boolean quoted = type == QueryColumnType.VARCHAR || type == QueryColumnType.CHAR || type == QueryColumnType.TIME || type == QueryColumnType.DATE + || type == QueryColumnType.TIMESTAMP || QueryColumnType.OBJECT == type || QueryColumnType.OTHER == type; + if ( quoted ) { + SQLWithParamValues.append( "'" ); + } + SQLWithParamValues.append( StringCaster.attempt( value ).getOrSupply( () -> String.valueOf( value ) ) ); + if ( quoted ) { + SQLWithParamValues.append( "'" ); } } /** * Apply query options to the provided {@link Statement} instance. *

    - * Any query options which pass through to the JDBC Statement interface will be applied here. This includes `queryTimeout`, `maxRows`, and `fetchSize`. + * Any query options which pass through to the JDBC Statement interface will be + * applied here. This includes `queryTimeout`, `maxRows`, and `fetchSize`. */ private void applyStatementOptions( Statement statement ) throws SQLException { IStruct options = this.queryOptions.toStruct(); @@ -520,7 +764,8 @@ private void applyStatementOptions( Statement statement ) throws SQLException { /** * TODO: Implement the following options: * ormoptions - * username and password : To evaluate later due to security concerns of overriding datasources, not going to implement unless requested + * username and password : To evaluate later due to security concerns of + * overriding datasources, not going to implement unless requested * clientInfo : Part of the connection: get/setClientInfo() */ } diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java index 028768ad3..61f678a57 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/QueryParameter.java @@ -90,13 +90,9 @@ private QueryParameter( String name, IStruct param ) { // allow nulls and null this.isNullParam = BooleanCaster.cast( param.getOrDefault( Key.nulls, param.getOrDefault( Key.nulls2, false ) ) ); this.isListParam = BooleanCaster.cast( param.getOrDefault( Key.list, false ) ); - String separator = StringCaster.cast( param.getOrDefault( Key.separator, "," ) ); - // Allow a name key in the struct if passed as an array of structs - if ( name == null ) { - name = ( String ) param.getAsString( Key._NAME ); - } + String separator = StringCaster.cast( param.getOrDefault( Key.separator, "," ) ); - Object v = param.get( Key.value ); + Object v = param.get( Key.value ); if ( this.isListParam ) { if ( v instanceof Array ) { // do nothing? diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java index e71e11ad6..bbe9b4257 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQExecutionService.java @@ -73,7 +73,7 @@ public static SQLNode parseSQL( String sql ) { DynamicObject trans = frTransService.startTransaction( "BL QoQ Parse", "" ); SQLParser parser = new SQLParser(); ParsingResult result; - + // System.out.println( "Parsing SQL: " + sql ); try { result = parser.parse( sql ); } finally { diff --git a/src/test/java/external/specs/QoQListsTest.cfc b/src/test/java/external/specs/QoQListsTest.cfc index a79e72aaa..47573d9c9 100644 --- a/src/test/java/external/specs/QoQListsTest.cfc +++ b/src/test/java/external/specs/QoQListsTest.cfc @@ -49,53 +49,7 @@ component extends="testbox.system.BaseSpec"{ }); describe( title='using param list=true' , body=function() { - describe( title='with new Query()' , body=function() { - beforeEach( function( currentSpec ) { - q = new Query( - dbtype = 'query', - queryWithDataIn = variables.queryWithDataIn - ); - }); - - it( title='when using numeric params' , body=function( currentSpec ) { - q.addParam( name: 'needle' , value: interestingNumbersAsAList , sqltype: 'numeric' , list: true ); - var actual = q.execute( sql = " - SELECT - id, - value - FROM queryWithDataIn - WHERE id IN ( :needle ) - " ).getResult(); - - expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); - }); - - it( title='when using numeric params and a custom separator' , body=function( currentSpec ) { - q.addParam( name: 'needle' , value: Replace( interestingNumbersAsAList , ',' , '|' ) , sqltype: 'numeric' , list: true , separator: '|' ); - var actual = q.execute( sql = " - SELECT - id, - value - FROM queryWithDataIn - WHERE id IN ( :needle ) - " ).getResult(); - - expect( actual.RecordCount ).toBe( ListLen( interestingNumbersAsAList , ',' ) ); - }); - - it( title='when using string params' , body=function( currentSpec ) { - q.addParam( name: 'needle' , value: interestingStringsAsAList , sqltype: 'varchar' , list: true ); - var actual = q.execute( sql = " - SELECT - id, - value - FROM queryWithDataIn - WHERE value IN ( :needle ) - " ).getResult(); - expect( actual.RecordCount ).toBe( ListLen( interestingStringsAsAList , ',' ) ); - }); - }); - + describe( title='with query{} ( cfquery )' , body=function() { it( title='when using numeric params' , body=function( currentSpec ) { query diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 84fbc1463..3665fc9a4 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -19,9 +19,9 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import ortus.boxlang.compiler.parser.BoxSourceType; import ortus.boxlang.compiler.parser.ParsingResult; import ortus.boxlang.compiler.parser.SQLParser; import ortus.boxlang.runtime.BoxRuntime; @@ -68,12 +68,10 @@ public void testMetadataVisitor() { where true = true order by t.baz limit 5 - """ - ); + """ ); IStruct data = Struct.of( "file", null, - "result", result - ); + "result", result ); BoxRuntime.getInstance().announce( "onParse", data ); assertThat( result ).isNotNull(); @@ -566,20 +564,27 @@ public void testNullAggregate() { public void testsdf() { instance.executeSource( """ - q = queryNew("id","numeric",[[1]]); - - queryExecute(" - select id - from q - where id= :id - ", - { - 'id': {type="integer", value=""} - }, - {dbtype:"query"} - ); - """, - context ); + queryWithDataIn = QueryNew('id,value', 'integer,varchar',[[1,'a'],[2,'b'],[3,'c'],[4,'d'],[5,'e']]); + actual = QueryExecute( + params = [ + { value: 3 }, + { value: '3,4' , sqltype: 'numeric' , list = true } + ], + options = { + dbtype: 'query' + }, + sql = " + SELECT + id, + value + FROM queryWithDataIn + WHERE id = ? + and id IN ( ? ) + " ); + + println(actual) + """, + context, BoxSourceType.CFSCRIPT ); } } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java index aa293f24e..76f920a17 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/BaseJDBCTest.java @@ -83,7 +83,8 @@ public static void setUp() { "host", "localhost", "port", "3306", "driver", "mysql", - "database", "mysqlDB" + "database", "mysqlDB", + "custom", "allowMultiQueries=true" ) ); instance.getConfiguration().datasources.put( mysqlName, diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java index c3ac5d6ba..e6fb938b7 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java @@ -110,6 +110,34 @@ public void testTimestampParamCompare() { assertEquals( 1, query.size() ); } + @EnabledIf( "tools.JDBCTestUtils#hasMySQLModule" ) + @DisplayName( "It can execute multiple statements in a single queryExecute() call" ) + @Test + public void testMultipleStatements() { + assertDoesNotThrow( () -> instance.executeStatement( + """ + result = queryExecute( ' + TRUNCATE TABLE developers; + INSERT INTO developers (id) VALUES (111); + INSERT INTO developers (id) VALUES (222); + SELECT * FROM developers; + INSERT INTO developers (id) VALUES (333); + INSERT INTO developers (id) VALUES (444); + ', + [], + { "datasource" : "mysqldatasource" } + ); + """, context ) + ); + Object multiStatementQueryReturn = variables.get( Key.of( "result" ) ); + assertThat( multiStatementQueryReturn ).isInstanceOf( Query.class ); + assertEquals( 2, ( ( Query ) multiStatementQueryReturn ).size(), "For compatibility, the last result should be returned" ); + + Query newTableRows = ( Query ) instance + .executeStatement( "queryExecute( 'SELECT * FROM developers WHERE id IN (111,222)', [],{ 'datasource' : 'mysqldatasource' } );", context ); + assertEquals( 2, newTableRows.size() ); + } + @DisplayName( "It can execute a query with no bindings on the default datasource" ) @Test public void testSimpleExecute() { @@ -328,7 +356,7 @@ public void testMissingStructBinding() { """, context ) ); - assertThat( e.getMessage() ).isEqualTo( "Missing param in query: [id]. SQL: SELECT * FROM developers WHERE id = :id" ); + assertThat( e.getMessage() ).isEqualTo( "Named parameter [:id] not provided to query." ); assertNull( variables.get( result ) ); } @@ -505,7 +533,7 @@ public void testResultVariable() { IStruct result = StructCaster.cast( resultObject ); assertThat( result ).containsKey( Key.sql ); - assertEquals( "SELECT * FROM developers WHERE role = ?", result.getAsString( Key.sql ) ); + assertEquals( "SELECT * FROM developers WHERE role = 'Developer'", result.getAsString( Key.sql ) ); assertThat( result ).containsKey( Key.sqlParameters ); assertEquals( Array.of( "Developer" ), result.getAsArray( Key.sqlParameters ) ); @@ -643,28 +671,6 @@ public void testGeneratedKey() { assertThat( DoubleCaster.cast( meta.get( Key.generatedKey ), false ) ).isEqualTo( 1.0d ); } - @DisplayName( "It can execute multiple statements in a single queryExecute() call like Lucee" ) - @Test - public void testMultipleStatements() { - // ACF 2023 will throw an error on this type of fooferall, but Lucee is fine with it and IMHO we should support it. - assertDoesNotThrow( () -> instance.executeStatement( - """ - result = queryExecute( ' - SELECT * FROM developers; - INSERT INTO developers (id) VALUES (111); - INSERT INTO developers (id) VALUES (222) - ' - ); - """, context ) - ); - Object multiStatementQueryReturn = variables.get( Key.of( "result" ) ); - assertThat( multiStatementQueryReturn ).isInstanceOf( Query.class ); - assertEquals( 4, ( ( Query ) multiStatementQueryReturn ).size(), "For compatibility, only the first result should be returned" ); - - Query newTableRows = ( Query ) instance.executeStatement( "queryExecute( 'SELECT * FROM developers WHERE id IN (111,222)' );", context ); - assertEquals( 2, newTableRows.size() ); - } - @DisplayName( "It can return cached query results within the cache timeout" ) @Test public void testQueryCaching() { diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java index d7694e0c7..4397a7eb8 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java @@ -391,7 +391,7 @@ public void testResultVariable() { IStruct result = StructCaster.cast( resultObject ); assertThat( result ).containsKey( Key.sql ); - assertEquals( "SELECT * FROM developers WHERE role = ?", result.getAsString( Key.sql ) ); + assertEquals( "SELECT * FROM developers WHERE role = 'Developer'", result.getAsString( Key.sql ) ); assertThat( result ).containsKey( Key.cached ); assertThat( result.getAsBoolean( Key.cached ) ).isEqualTo( false ); diff --git a/src/test/java/ortus/boxlang/runtime/jdbc/DataSourceTest.java b/src/test/java/ortus/boxlang/runtime/jdbc/DataSourceTest.java index 3e2d3f6bd..ad990e26a 100644 --- a/src/test/java/ortus/boxlang/runtime/jdbc/DataSourceTest.java +++ b/src/test/java/ortus/boxlang/runtime/jdbc/DataSourceTest.java @@ -238,7 +238,7 @@ void testDatasourceWithMissingNamedParams() { context ); } ); - assertEquals( "Missing param in query: [name]. SQL: SELECT * FROM developers WHERE name = :name", exception.getMessage() ); + assertEquals( "Named parameter [:name] not provided to query.", exception.getMessage() ); } catch ( SQLException e ) { throw new RuntimeException( e ); } From 4dacebf112efbf03ce4e9f0b80e59cd0ed6af87d Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 1 Jan 2025 13:01:06 -0600 Subject: [PATCH 084/161] missing import --- src/test/java/ortus/boxlang/compiler/QoQParseTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java index 3665fc9a4..0ff7236ce 100644 --- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java +++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import ortus.boxlang.compiler.parser.BoxSourceType; From 798343c6a6223a0ae3d7ae2aa8ac8c0f036fc5e3 Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Sat, 4 Jan 2025 14:20:38 -0500 Subject: [PATCH 085/161] BL-899 Resolve --- .../ortus/boxlang/runtime/util/LocalizationUtil.java | 2 +- .../runtime/bifs/global/format/NumberFormatTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/util/LocalizationUtil.java b/src/main/java/ortus/boxlang/runtime/util/LocalizationUtil.java index e70498d05..f2190bf12 100644 --- a/src/main/java/ortus/boxlang/runtime/util/LocalizationUtil.java +++ b/src/main/java/ortus/boxlang/runtime/util/LocalizationUtil.java @@ -205,7 +205,7 @@ public final class LocalizationUtil { NUMBER_FORMAT_PATTERNS.put( Key.of( "+" ), "+0;-0" ); NUMBER_FORMAT_PATTERNS.put( Key.of( "-" ), " 0;-0" ); NUMBER_FORMAT_PATTERNS.put( Key.dollarFormat, "$#,##0.00;($#,##0.00)" ); - NUMBER_FORMAT_PATTERNS.put( DEFAULT_NUMBER_FORMAT_KEY, "#,#00.#" ); + NUMBER_FORMAT_PATTERNS.put( DEFAULT_NUMBER_FORMAT_KEY, "#,##0.#" ); } public static final String CURRENCY_TYPE_LOCAL = "local"; diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/format/NumberFormatTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/format/NumberFormatTest.java index 02f7bbb76..2f68fe39c 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/format/NumberFormatTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/format/NumberFormatTest.java @@ -261,4 +261,16 @@ public void testLocaleDefault() { assertEquals( variables.getAsString( result ), "12,345" ); } + // https://ortussolutions.atlassian.net/browse/BL-899 + @DisplayName( "It tests the BIF LSNumberFormat will not add leading zeroes to int < 10" ) + @Test + public void testLeadingZeros() { + instance.executeSource( + """ + result = LSnumberFormat( 1 ); + """, + context ); + assertEquals( "1", variables.getAsString( result ) ); + } + } From 85acd302bb0bf445853f62202ef43eba74796a26 Mon Sep 17 00:00:00 2001 From: Jacob Beers Date: Sat, 4 Jan 2025 20:49:31 -0600 Subject: [PATCH 086/161] BL-904 Fix empty script execution --- .../boxlang/compiler/asmboxpiler/AsmHelper.java | 7 +++++-- .../ortus/boxlang/runtime/BoxRuntimeTest.java | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmHelper.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmHelper.java index 036a72a0a..05982e27b 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmHelper.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmHelper.java @@ -898,9 +898,12 @@ public static void methodWithContextAndClassLocator( ClassNode classNode, false ); tracker.storeNewVariable( Opcodes.ASTORE ).nodes().forEach( ( node ) -> node.accept( methodVisitor ) ); - supplier.get().forEach( node -> node.accept( methodVisitor ) ); + var nodes = supplier.get(); - if ( implicityReturnNull && !returnType.equals( Type.VOID_TYPE ) ) { + nodes.forEach( node -> node.accept( methodVisitor ) ); + + if ( ( implicityReturnNull && !returnType.equals( Type.VOID_TYPE ) ) + || ( nodes.size() == 0 && !returnType.equals( Type.VOID_TYPE ) ) ) { // push a null onto the stack so that we can return it if there isn't an explicity return methodVisitor.visitInsn( Opcodes.ACONST_NULL ); } diff --git a/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java b/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java index af2edb43b..de70c5ebb 100644 --- a/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java +++ b/src/test/java/ortus/boxlang/runtime/BoxRuntimeTest.java @@ -168,6 +168,22 @@ public void testItCanExecuteMoreStatements() { } + @DisplayName( "It can execute an empty script" ) + @Test + public void testItCanRunAnEmptyScript() { + + BoxRuntime instance = BoxRuntime.getInstance( true ); + IBoxContext context = new ScriptingRequestBoxContext(); + + instance.executeSource( + """ + // Testing code here + """, + context + ); + + } + @DisplayName( "It can get the default file extensions" ) @Test public void testItCanGetTheDefaultFileExtensions() { From 8836e608b3bc2399845f2e7a0dcb1acbd6cf998f Mon Sep 17 00:00:00 2001 From: Michael Born Date: Mon, 6 Jan 2025 08:42:13 -0500 Subject: [PATCH 087/161] JDBC - Fix autoCommit reset on transaction close HikariCP does this automatically upon transaction close, but will log a warning if they notice the connection autoCommit mode does not match the pool. And taking care of it ourselves is just good cleanup. See https://stackoverflow.com/questions/41202242/reset-autocommit-on-connection-in-hikaricp --- src/main/java/ortus/boxlang/runtime/jdbc/Transaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/Transaction.java b/src/main/java/ortus/boxlang/runtime/jdbc/Transaction.java index afbfa5bc8..98011ccd8 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/Transaction.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/Transaction.java @@ -298,7 +298,7 @@ public Transaction end() { ); announce( BoxEvent.ON_TRANSACTION_RELEASE, releaseEventData ); - if ( this.connection.getAutoCommit() ) { + if ( !this.connection.getAutoCommit() ) { this.connection.setAutoCommit( true ); } From c75bc41e8ba912095ac379cab050e3778017ec37 Mon Sep 17 00:00:00 2001 From: Michael Born Date: Mon, 6 Jan 2025 08:53:38 -0500 Subject: [PATCH 088/161] JDBC - Ensure connectionManager shutdown closes default datasource AND ends active transactions There SHOULD never be a case where transaction is still open at this point, however... it's good to code defensively. --- .../boxlang/runtime/jdbc/ConnectionManager.java | 14 +++++++++++++- .../runtime/components/jdbc/TransactionTest.java | 12 +++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java b/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java index 70f58d327..6dff0a813 100644 --- a/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java +++ b/src/main/java/ortus/boxlang/runtime/jdbc/ConnectionManager.java @@ -203,7 +203,9 @@ public ITransaction beginTransaction( DataSource datasource ) { } /** - * Set the active transaction for this request/thread/BoxLang context. + * Close the active transaction for this request/thread/BoxLang context. + * + * In case of nested transactions, will close the inner transaction and update the reference to the parent transaction. Otherwise will close the outer (only) transaction and nullify the reference. */ public ConnectionManager endTransaction() { this.transaction.end(); @@ -599,6 +601,16 @@ public void shutdown() { datasource.shutdown(); } ); this.datasources.clear(); + + if ( this.defaultDatasource != null ) { + this.defaultDatasource.shutdown(); + this.defaultDatasource = null; + } + + if ( this.transaction != null ) { + this.transaction.end(); + this.transaction = null; + } } /** diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/TransactionTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/TransactionTest.java index 5793107bd..dff9ecd26 100644 --- a/src/test/java/ortus/boxlang/runtime/components/jdbc/TransactionTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/TransactionTest.java @@ -19,18 +19,21 @@ package ortus.boxlang.runtime.components.jdbc; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import static com.google.common.truth.Truth.assertThat; + import ortus.boxlang.runtime.bifs.global.jdbc.BaseJDBCTest; +import ortus.boxlang.runtime.context.IJDBCCapableContext; import ortus.boxlang.runtime.events.BoxEvent; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Query; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.types.exceptions.DatabaseException; -import static org.junit.jupiter.api.Assertions.*; - /** * Tests the basics of the transaction component, especially attribute validation. *

    @@ -121,6 +124,9 @@ public void testTransactionEndsOnException() { Query theResult = ( Query ) getInstance() .executeStatement( "queryExecute( 'SELECT * FROM developers WHERE id IN (111)' );", getContext() ); assertThat( theResult.size() ).isEqualTo( 1 ); + + // The connection manager won't close connections if it thinks they are from an active transaction. + assertThat( ( ( IJDBCCapableContext ) getContext() ).getConnectionManager().isInTransaction() ).isFalse(); } @DisplayName( "Emits transactional events" ) From 65e8bc57443ed264e1941cbdd464cf460dd37072 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:50:59 +0000 Subject: [PATCH 089/161] Bump ch.qos.logback:logback-classic from 1.5.15 to 1.5.16 Bumps [ch.qos.logback:logback-classic](https://github.com/qos-ch/logback) from 1.5.15 to 1.5.16. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.15...v_1.5.16) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-classic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3f9123928..7a926c348 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ dependencies { // https://mvnrepository.com/artifact/org.slf4j/slf4j-api implementation 'org.slf4j:slf4j-api:2.0.16' // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic - implementation 'ch.qos.logback:logback-classic:1.5.15' + implementation 'ch.qos.logback:logback-classic:1.5.16' // https://mvnrepository.com/artifact/com.zaxxer/HikariCP implementation 'com.zaxxer:HikariCP:6.2.1' // https://mvnrepository.com/artifact/org.ow2.asm/asm-tree From e0e46db17b1e3b1af8e2f6ec15f6c12d59339c7d Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 16:35:06 -0600 Subject: [PATCH 090/161] BL-908 --- .github/workflows/tests.yml | 5 ++ .gitignore | 1 + .../java/ortus/boxlang/compiler/Boxpiler.java | 8 +- .../ortus/boxlang/compiler/ClassInfo.java | 6 +- .../ortus/boxlang/compiler/DiskClassUtil.java | 14 +++ .../compiler/asmboxpiler/ASMBoxpiler.java | 5 ++ .../compiler/javaboxpiler/JavaBoxpiler.java | 90 +++++++++++++++++-- .../bifs/global/system/SystemExecute.java | 40 +++++---- .../runtime/loader/DiskClassLoader.java | 4 +- src/test/java/external/TestBoxTest.java | 12 ++- 10 files changed, 152 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cdee55451..080548877 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,6 +46,11 @@ jobs: with: gradle-version: "8.7" + - name: Install TestBox + - uses: Ortus-Solutions/commandbox-action@v1.0.3 + with: + cmd: install testbox@be src/test/resources --verbose --noSave + # - name: Setup Database and Fixtures # run: | # sudo systemctl start mysql.service diff --git a/.gitignore b/.gitignore index cef2026c1..baf99aeb1 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ derby.log src/test/java/ortus/boxlang/compiler/Precompiled.bx src/test/java/ortus/boxlang/compiler/Precompiled.bxs +/src/test/resources/testbox diff --git a/src/main/java/ortus/boxlang/compiler/Boxpiler.java b/src/main/java/ortus/boxlang/compiler/Boxpiler.java index 75d67de9e..678e0c3d1 100644 --- a/src/main/java/ortus/boxlang/compiler/Boxpiler.java +++ b/src/main/java/ortus/boxlang/compiler/Boxpiler.java @@ -231,7 +231,9 @@ public Class compileTemplate( ResolvedFilePath resolvedFilePath ) var classPool = getClassPool( classInfo.classPoolName() ); classPool.putIfAbsent( classInfo.fqn().toString(), classInfo ); // If the new class is newer than the one on disk, recompile it - if ( classPool.get( classInfo.fqn().toString() ).lastModified() < classInfo.lastModified() ) { + long lastModified = classPool.get( classInfo.fqn().toString() ).lastModified(); + long lastModified2 = classInfo.lastModified(); + if ( ( lastModified > 0 ) && ( lastModified2 > 0 ) && ( lastModified != lastModified2 ) ) { try { // Don't know if this does anything, but calling it for good measure classPool.get( classInfo.fqn().toString() ).getClassLoader().close(); @@ -276,7 +278,9 @@ public Class compileClass( ResolvedFilePath resolvedFilePath ) { var classPool = getClassPool( classInfo.classPoolName() ); classPool.putIfAbsent( classInfo.fqn().toString(), classInfo ); // If the new class is newer than the one on disk, recompile it - if ( classPool.get( classInfo.fqn().toString() ).lastModified() < classInfo.lastModified() ) { + long lastModified = classPool.get( classInfo.fqn().toString() ).lastModified(); + long lastModified2 = classInfo.lastModified(); + if ( ( lastModified > 0 ) && ( lastModified2 > 0 ) && ( lastModified != lastModified2 ) ) { try { // Don't know if this does anything, but calling it for good measure classPool.get( classInfo.fqn().toString() ).getClassLoader().close(); diff --git a/src/main/java/ortus/boxlang/compiler/ClassInfo.java b/src/main/java/ortus/boxlang/compiler/ClassInfo.java index 4e7fe4708..77f03c455 100644 --- a/src/main/java/ortus/boxlang/compiler/ClassInfo.java +++ b/src/main/java/ortus/boxlang/compiler/ClassInfo.java @@ -24,7 +24,7 @@ public record ClassInfo( String returnType, BoxSourceType sourceType, String source, - Long lastModified, + long lastModified, DiskClassLoader[] diskClassLoader, InterfaceProxyDefinition interfaceProxyDefinition, IBoxpiler boxpiler, @@ -90,7 +90,7 @@ public static ClassInfo forTemplate( ResolvedFilePath resolvedFilePath, BoxSourc "void", sourceType, null, - Math.min( resolvedFilePath.absolutePath().toFile().lastModified(), System.currentTimeMillis() ), + resolvedFilePath.absolutePath().toFile().lastModified(), new DiskClassLoader[ 1 ], null, boxpiler, @@ -106,7 +106,7 @@ public static ClassInfo forClass( ResolvedFilePath resolvedFilePath, BoxSourceTy null, sourceType, null, - Math.min( resolvedFilePath.absolutePath().toFile().lastModified(), System.currentTimeMillis() ), + resolvedFilePath.absolutePath().toFile().lastModified(), new DiskClassLoader[ 1 ], null, boxpiler, diff --git a/src/main/java/ortus/boxlang/compiler/DiskClassUtil.java b/src/main/java/ortus/boxlang/compiler/DiskClassUtil.java index 611144858..08c48c8e4 100644 --- a/src/main/java/ortus/boxlang/compiler/DiskClassUtil.java +++ b/src/main/java/ortus/boxlang/compiler/DiskClassUtil.java @@ -125,10 +125,24 @@ public void writeJavaSource( String classPoolName, String fqn, String javaSource * @param bytes The bytes to write */ public void writeBytes( String classPoolName, String fqn, String extension, byte[] bytes ) { + writeBytes( classPoolName, fqn, extension, bytes, 0L ); + } + + /** + * Write a file to the directory configured for BoxLang + * + * @param fqn The fully qualified name of the class + * @param extension The extension of the file + * @param bytes The bytes to write + */ + public void writeBytes( String classPoolName, String fqn, String extension, byte[] bytes, long lastModifiedDate ) { Path diskPath = generateDiskpath( classPoolName, fqn, extension ); diskPath.toFile().getParentFile().mkdirs(); try { Files.write( diskPath, bytes ); + if ( lastModifiedDate > 0 ) { + diskPath.toFile().setLastModified( lastModifiedDate ); + } } catch ( IOException e ) { throw new BoxRuntimeException( "Unable to write Java Sourece file to disk", e ); } diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/ASMBoxpiler.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/ASMBoxpiler.java index e8a2b3ca9..625c992be 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/ASMBoxpiler.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/ASMBoxpiler.java @@ -22,6 +22,7 @@ import ortus.boxlang.compiler.ast.visitor.QueryEscapeSingleQuoteVisitor; import ortus.boxlang.compiler.parser.Parser; import ortus.boxlang.compiler.parser.ParsingResult; +import ortus.boxlang.runtime.BoxRuntime; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; import ortus.boxlang.runtime.util.ResolvedFilePath; @@ -75,6 +76,10 @@ public void printTranspiledCode( ParsingResult result, ClassInfo classInfo, Prin @Override public void compileClassInfo( String classPoolName, String FQN ) { + if ( BoxRuntime.getInstance().inDebugMode() ) { + // Some debugging to help testing + System.out.println( "ASM BoxPiler Compiling " + FQN ); + } ClassInfo classInfo = getClassPool( classPoolName ).get( FQN ); if ( classInfo == null ) { throw new BoxRuntimeException( "ClassInfo not found for " + FQN ); diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/JavaBoxpiler.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/JavaBoxpiler.java index d1ffba881..9cc3c59d5 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/JavaBoxpiler.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/JavaBoxpiler.java @@ -18,7 +18,9 @@ package ortus.boxlang.compiler.javaboxpiler; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintStream; import java.net.URISyntaxException; import java.nio.file.Path; @@ -32,8 +34,12 @@ import java.util.stream.Collectors; import javax.tools.DiagnosticCollector; +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; import javax.tools.JavaCompiler; +import javax.tools.JavaFileManager; import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; import javax.tools.StandardJavaFileManager; import javax.tools.StandardLocation; import javax.tools.ToolProvider; @@ -172,7 +178,7 @@ public String generateJavaSource( BoxNode node, ClassInfo classInfo ) { public void compileClassInfo( String classPoolName, String FQN ) { if ( BoxRuntime.getInstance().inDebugMode() ) { // Some debugging to help testing - System.out.println( "Compiling " + FQN ); + System.out.println( "Java BoxPiler Compiling " + FQN ); } ClassInfo classInfo = getClassPool( classPoolName ).get( FQN ); if ( classInfo == null ) { @@ -186,12 +192,12 @@ public void compileClassInfo( String classPoolName, String FQN ) { return; } ParsingResult result = parseOrFail( sourceFile ); - compileSource( generateJavaSource( result.getRoot(), classInfo ), classInfo.fqn().toString(), classPoolName ); + compileSource( generateJavaSource( result.getRoot(), classInfo ), classInfo.fqn().toString(), classPoolName, classInfo.lastModified() ); } else if ( classInfo.source() != null ) { ParsingResult result = parseOrFail( classInfo.source(), classInfo.sourceType(), classInfo.isClass() ); - compileSource( generateJavaSource( result.getRoot(), classInfo ), classInfo.fqn().toString(), classPoolName ); + compileSource( generateJavaSource( result.getRoot(), classInfo ), classInfo.fqn().toString(), classPoolName, classInfo.lastModified() ); } else if ( classInfo.interfaceProxyDefinition() != null ) { - compileSource( generateProxyJavaSource( classInfo ), classInfo.fqn().toString(), classPoolName ); + compileSource( generateProxyJavaSource( classInfo ), classInfo.fqn().toString(), classPoolName, classInfo.lastModified() ); } else { throw new BoxRuntimeException( "Unknown class info type: " + classInfo.toString() ); } @@ -204,7 +210,7 @@ public void compileClassInfo( String classPoolName, String FQN ) { * @param fqn The fully qualified name of the class */ @SuppressWarnings( "unused" ) - private void compileSource( String javaSource, String fqn, String classPoolName ) { + private void compileSource( String javaSource, String fqn, String classPoolName, Long lastModifiedDate ) { DynamicObject trans = frTransService.startTransaction( "Java Compilation", fqn ); // This is just for debugging. Remove later. @@ -214,12 +220,45 @@ private void compileSource( String javaSource, String fqn, String classPoolName DiagnosticCollector diagnostics = new DiagnosticCollector<>(); // Get the standard file manager - StandardJavaFileManager fileManager = compiler.getStandardFileManager( diagnostics, null, null ); - + StandardJavaFileManager standardFileManager = compiler.getStandardFileManager( diagnostics, null, null ); // Set the location where .class files should be written String classPoolDiskPrefix = RegexBuilder.of( classPoolName, RegexBuilder.NON_ALPHANUMERIC ).replaceAllAndGet( "_" ); - fileManager.setLocation( StandardLocation.CLASS_OUTPUT, Arrays.asList( classGenerationDirectory.resolve( classPoolDiskPrefix ).toFile() ) ); + standardFileManager.setLocation( StandardLocation.CLASS_OUTPUT, Arrays.asList( classGenerationDirectory.resolve( classPoolDiskPrefix ).toFile() ) ); + JavaFileManager fileManager = new ForwardingJavaFileManager( standardFileManager ) { + + @Override + public JavaFileObject getJavaFileForOutput( Location location, String className, + JavaFileObject.Kind kind, FileObject sibling ) throws IOException { + SimpleJavaFileObject d; + JavaFileObject f = this.fileManager.getJavaFileForOutput( location, className, + kind, sibling ); + return new SimpleJavaFileObject( f.toUri(), kind ) { + + @Override + public OutputStream openOutputStream() + throws IOException { + return new FileOutputStream( + f.toUri().getPath() ) { + + @Override + public void close() + throws IOException { + super.close(); + if ( lastModifiedDate > 0 ) { + Paths + .get( + f.toUri() ) + .toFile() + .setLastModified( + lastModifiedDate ); + } + } + }; + } + }; + } + }; String javaRT = System.getProperty( "java.class.path" ); String jarPath = Paths.get( getClass().getProtectionDomain().getCodeSource().getLocation().toURI() ).toString(); @@ -327,4 +366,39 @@ public List compileTemplateBytes( ResolvedFilePath resolvedFilePath ) { return diskClassUtil.readClassBytes( classInfo.classPoolName(), classInfo.fqn().toString() ); } + /* + * private static class CustomOutputStream extends OutputStream { + * + * private final OutputStream delegate; + * private final Path path; + * private final Instant customModifiedDate; + * + * public CustomOutputStream( OutputStream delegate, Path path, Instant customModifiedDate ) { + * this.delegate = delegate; + * this.path = path; + * this.customModifiedDate = customModifiedDate; + * } + * + * @Override + * public void write( int b ) throws IOException { + * delegate.write( b ); + * } + * + * @Override + * public void write( byte[] b ) throws IOException { + * delegate.write( b ); + * } + * + * @Override + * public void write( byte[] b, int off, int len ) throws IOException { + * delegate.write( b, off, len ); + * } + * + * @Override + * public void close() throws IOException { + * delegate.close(); + * Files.setLastModifiedTime( path, FileTime.from( customModifiedDate ) ); + * } + * } + */ } diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/SystemExecute.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/SystemExecute.java index b454ed6b2..f238d89fc 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/SystemExecute.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/SystemExecute.java @@ -34,6 +34,7 @@ import ortus.boxlang.runtime.bifs.BoxBIF; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.dynamic.casters.ArrayCaster; +import ortus.boxlang.runtime.dynamic.casters.CastAttempt; import ortus.boxlang.runtime.dynamic.casters.StringCaster; import ortus.boxlang.runtime.scopes.ArgumentsScope; import ortus.boxlang.runtime.scopes.Key; @@ -113,21 +114,30 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { } ); cmd.add( bin ); - if ( args instanceof String ) { - // ensure we preserve any spaces in - Matcher matches = Pattern.compile( "[^\\s\"']+|\"[^\"]*\"|'[^']*'" ).matcher( StringCaster.cast( args ) ); - matches.reset(); - while ( matches.find() ) - cmd.add( matches.group() ); - } else if ( args instanceof Array ) { - ArrayCaster.cast( args ).stream().forEach( arg -> cmd.add( StringCaster.cast( arg ) ) ); - } else { - throw new BoxRuntimeException( - String.format( - "The provided process arguments provided [%s] could not be parsed in to command arguments", - args.toString() - ) - ); + if ( args != null ) { + CastAttempt strAttempt = StringCaster.attempt( args ); + // If the args are a simple value, parse out each argument + if ( strAttempt.wasSuccessful() ) { + // ensure we preserve any spaces in + Matcher matches = Pattern.compile( "[^\\s\"']+|\"[^\"]*\"|'[^']*'" ).matcher( strAttempt.get() ); + matches.reset(); + while ( matches.find() ) { + cmd.add( matches.group() ); + } + } else { + // If args are an array, use the array values as arguments directly + CastAttempt arrAttempt = ArrayCaster.attempt( args ); + if ( arrAttempt.wasSuccessful() ) { + arrAttempt.get().stream().forEach( arg -> cmd.add( StringCaster.cast( arg ) ) ); + } else { + throw new BoxRuntimeException( + String.format( + "The provided process arguments provided [%s] could not be parsed in to command arguments", + args.toString() + ) + ); + } + } } ProcessBuilder processBuilder = new ProcessBuilder( cmd ); diff --git a/src/main/java/ortus/boxlang/runtime/loader/DiskClassLoader.java b/src/main/java/ortus/boxlang/runtime/loader/DiskClassLoader.java index a1f78d153..d4c27533b 100644 --- a/src/main/java/ortus/boxlang/runtime/loader/DiskClassLoader.java +++ b/src/main/java/ortus/boxlang/runtime/loader/DiskClassLoader.java @@ -135,7 +135,7 @@ private boolean needsCompile( ClassInfo classInfo, Path diskPath, String name, S // There is a class file cached on disk if ( hasClass( diskPath ) ) { // If the class file is older than the source file - if ( classInfo != null && classInfo.lastModified() > diskPath.toFile().lastModified() ) { + if ( classInfo != null && classInfo.lastModified() > 0 && classInfo.lastModified() != diskPath.toFile().lastModified() ) { return true; } return false; @@ -174,7 +174,7 @@ public boolean hasClass( String name, long lastModified ) { return false; } // If source file is modified after class file - if ( lastModified > diskPath.toFile().lastModified() ) { + if ( lastModified > 0 && lastModified != diskPath.toFile().lastModified() ) { return false; } diff --git a/src/test/java/external/TestBoxTest.java b/src/test/java/external/TestBoxTest.java index 6e37db8e5..4d42b8612 100644 --- a/src/test/java/external/TestBoxTest.java +++ b/src/test/java/external/TestBoxTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; @@ -61,13 +60,20 @@ public void setupEach() { } @TestFactory - @Disabled Stream runDynamicTests() { // @formatter:off instance.executeSource( """ - application name="testbox runner" mappings={"/testbox":expandPath("/src/test/resources/testbox")}; + testboxDir = expandPath("/src/test/resources/testbox"); + if( !directoryExists( testboxDir ) ) { + // Assumes CommandBox is installed. Avoid this by installing TestBox yourself via another means before running the tests + println( "Installing TestBox into: " & testboxDir ); + response = systemExecute( name = "box", arguments = [ "install", "testbox@be", "src/test/resources", "--verbose", "--noSave" ], timeout=60 ); + println( response ); + } + + application name="testbox runner" mappings={"/testbox":testboxDir}; result = new testbox.system.TestBox().runRaw( directory='src.test.java.external.specs' ).getMemento(); From 0768b5c43eca14aa3f9c1d5c077c0719336a88d7 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 16:37:09 -0600 Subject: [PATCH 091/161] Fix yaml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 080548877..6abc3f491 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,7 +47,7 @@ jobs: gradle-version: "8.7" - name: Install TestBox - - uses: Ortus-Solutions/commandbox-action@v1.0.3 + uses: Ortus-Solutions/commandbox-action@v1.0.3 with: cmd: install testbox@be src/test/resources --verbose --noSave From 1b915bfed007358cca511072f580696e00ddbd48 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 16:41:23 -0600 Subject: [PATCH 092/161] Use Commandbox container prior to java install --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6abc3f491..f7f12ef59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,11 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 + - name: Install TestBox + uses: Ortus-Solutions/commandbox-action@v1.0.3 + with: + cmd: install testbox@be src/test/resources --verbose --noSave + - name: Setup Java uses: actions/setup-java@v4 with: @@ -46,11 +51,6 @@ jobs: with: gradle-version: "8.7" - - name: Install TestBox - uses: Ortus-Solutions/commandbox-action@v1.0.3 - with: - cmd: install testbox@be src/test/resources --verbose --noSave - # - name: Setup Database and Fixtures # run: | # sudo systemctl start mysql.service From 4137e8ffca324359000b8f83e66d6967d8be23ea Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 16:50:06 -0600 Subject: [PATCH 093/161] Try installing CommandBox for stupid Windows to work --- .github/workflows/tests.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f7f12ef59..e4ca207bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,10 +35,6 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install TestBox - uses: Ortus-Solutions/commandbox-action@v1.0.3 - with: - cmd: install testbox@be src/test/resources --verbose --noSave - name: Setup Java uses: actions/setup-java@v4 @@ -57,8 +53,11 @@ jobs: # mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} -e 'CREATE DATABASE mementifier;' # mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} < test-harness/tests/resources/coolblog.sql - #- name: Setup CommandBox CLI - # uses: Ortus-Solutions/setup-commandbox@v2.0.1 + - name: Setup CommandBox CLI + uses: Ortus-Solutions/setup-commandbox@v2.0.1 + + - name: Install TestBox + run: box install testbox@be src/test/resources --verbose --noSave # Not Needed in this module #- name: Setup Environment For Testing Process From 1b79d7fcd15da9f58cdfbc9b85330b5e6080b364 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 20:07:40 -0600 Subject: [PATCH 094/161] Try choco for windows --- .github/workflows/tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e4ca207bb..288740869 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,9 +53,14 @@ jobs: # mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} -e 'CREATE DATABASE mementifier;' # mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} < test-harness/tests/resources/coolblog.sql - - name: Setup CommandBox CLI + - name: Setup CommandBox CLI Linux + if: ${{ matrix.os }} != 'windows-latest' uses: Ortus-Solutions/setup-commandbox@v2.0.1 + - name: Setup CommandBox CLI Windows + if: ${{ matrix.os }} == 'windows-latest' + run: choco install commandbox + - name: Install TestBox run: box install testbox@be src/test/resources --verbose --noSave From 2f3bb8bb3bdb0eecd4507a663b42c105a2215520 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 20:19:49 -0600 Subject: [PATCH 095/161] Try syntax variation --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 288740869..142abc766 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,11 +54,11 @@ jobs: # mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} < test-harness/tests/resources/coolblog.sql - name: Setup CommandBox CLI Linux - if: ${{ matrix.os }} != 'windows-latest' + if: ${{ matrix.os != 'windows-latest' }} uses: Ortus-Solutions/setup-commandbox@v2.0.1 - name: Setup CommandBox CLI Windows - if: ${{ matrix.os }} == 'windows-latest' + if: ${{ matrix.os == 'windows-latest' }} run: choco install commandbox - name: Install TestBox From 7d568622c75a355b8967bf9c188c8704db67e6ba Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Wed, 8 Jan 2025 20:38:30 -0600 Subject: [PATCH 096/161] Shorten testbox test names --- src/test/java/external/TestBoxTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/external/TestBoxTest.java b/src/test/java/external/TestBoxTest.java index 4d42b8612..2b67c6352 100644 --- a/src/test/java/external/TestBoxTest.java +++ b/src/test/java/external/TestBoxTest.java @@ -123,7 +123,7 @@ function mapSuites( array suiteStats ) { return acc.append( mapSuites( bundle.suiteStats ) // Slap the bundle name on the front of each spec from all the nested suites .map( s => { - s.name = bundle.path & ' - ' & s.name; + s.name = bundle.path.replace('src.test.java.external.specs.', '') & ' - ' & s.name; return s; } ), true ); }, [] ); From c811305008231f4dd485bab39b0f6e875c2f080d Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 9 Jan 2025 12:56:34 -0600 Subject: [PATCH 097/161] Add some application tests --- .../TestCases/phase3/ApplicationTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/test/java/TestCases/phase3/ApplicationTest.java b/src/test/java/TestCases/phase3/ApplicationTest.java index 69a04549e..2c26cddc8 100644 --- a/src/test/java/TestCases/phase3/ApplicationTest.java +++ b/src/test/java/TestCases/phase3/ApplicationTest.java @@ -295,4 +295,38 @@ public void testTimezoneDeclaration() { assertThat( zone.getId() ).isEqualTo( "America/Los_Angeles" ); } + @DisplayName( "Can update application without name" ) + @Test + public void testUpdateApplicationWithoutName() { + // @formatter:off + instance.executeSource( + """ + application + name="testUpdateApplicationWithoutName" + sessionmanagement="true"; + + firstSessionID = session.sessionID; + + newMappings = { + "/UpdateApplicationWithoutName" : "/src/test/resources/libs/" + } + + application + action ="update" + mappings ="#newMappings#"; + + secondSessionID = session.sessionID; + + result = GetApplicationMetadata(); + """, context ); + // @formatter:on + + IStruct result = variables.getAsStruct( Key.result ); + assertThat( result.get( Key._NAME ) ).isEqualTo( "testUpdateApplicationWithoutName" ); + assertThat( result.get( Key.mappings ) ).isNotNull(); + assertThat( result.get( Key.mappings ) ).isInstanceOf( IStruct.class ); + assertThat( result.getAsStruct( Key.mappings ).get( "/UpdateApplicationWithoutName" ) ).isEqualTo( "/src/test/resources/libs/" ); + assertThat( variables.get( Key.of( "firstSessionID" ) ) ).isEqualTo( variables.get( Key.of( "secondSessionID" ) ) ); + } + } From 02649bc8b7a161e3a30213ac977147e04ba4daa3 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 9 Jan 2025 13:22:20 -0600 Subject: [PATCH 098/161] BL-909 --- src/test/java/TestCases/phase1/CoreLangTest.java | 13 +++++++++++++ .../java/TestCases/phase1/TestBodyResultError.bx | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100644 src/test/java/TestCases/phase1/TestBodyResultError.bx diff --git a/src/test/java/TestCases/phase1/CoreLangTest.java b/src/test/java/TestCases/phase1/CoreLangTest.java index 4f1c513fa..4f40a882d 100644 --- a/src/test/java/TestCases/phase1/CoreLangTest.java +++ b/src/test/java/TestCases/phase1/CoreLangTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -4258,4 +4259,16 @@ public void testDateCOmpare() { assertThat( variables.getAsBoolean( result ) ).isFalse(); } + @Test + @Disabled( "BL-909" ) + public void testBodyResultError() { + // @formatter:off + instance.executeSource( + """ + new src.test.java.TestCases.phase1.TestBodyResultError(); + """, + context ); + // @formatter:on + } + } diff --git a/src/test/java/TestCases/phase1/TestBodyResultError.bx b/src/test/java/TestCases/phase1/TestBodyResultError.bx new file mode 100644 index 000000000..1b566eec1 --- /dev/null +++ b/src/test/java/TestCases/phase1/TestBodyResultError.bx @@ -0,0 +1,10 @@ +class { + + function foo() { + lock name="foo" throwontimeout=true timeout=5 { + return; + } + } + + foo(); +} \ No newline at end of file From 2eb1a96e375e7bd9bfa69dac1fbb599c742920ed Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 9 Jan 2025 14:37:12 -0600 Subject: [PATCH 099/161] BL-896 --- .../runtime/dynamic/casters/NumberCaster.java | 44 ++++++++++++++++++- .../ortus/boxlang/runtime/operators/Plus.java | 4 +- .../java/TestCases/phase1/OperatorsTest.java | 12 +++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java index 0ccdf8278..c6d81e681 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/NumberCaster.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.math.NumberUtils; import ortus.boxlang.runtime.interop.DynamicObject; +import ortus.boxlang.runtime.types.DateTime; import ortus.boxlang.runtime.types.exceptions.BoxCastException; import ortus.boxlang.runtime.types.util.MathUtil; @@ -34,6 +35,19 @@ public class NumberCaster implements IBoxCaster { public static boolean booleansAreNumbers = false; + /** + * Tests to see if the value can be cast to a Number. + * Returns a {@code CastAttempt} which will contain the result if casting was + * was successfull, or can be interogated to proceed otherwise. + * + * @param object The value to cast to a Number + * + * @return The Number value + */ + public static CastAttempt attempt( Object object, boolean castDates ) { + return CastAttempt.ofNullable( cast( object, false, castDates ) ); + } + /** * Tests to see if the value can be cast to a Number. * Returns a {@code CastAttempt} which will contain the result if casting was @@ -44,7 +58,7 @@ public class NumberCaster implements IBoxCaster { * @return The Number value */ public static CastAttempt attempt( Object object ) { - return CastAttempt.ofNullable( cast( object, false ) ); + return attempt( object, false ); } /** @@ -58,6 +72,17 @@ public static Number cast( Object object ) { return cast( object, true ); } + /** + * Used to cast anything to a Number, throwing exception if we fail + * + * @param object The value to cast to a Number + * + * @return The Number value + */ + public static Number cast( boolean castDates, Object object ) { + return cast( object, true, castDates ); + } + /** * Used to cast anything to a Number * @@ -67,6 +92,18 @@ public static Number cast( Object object ) { * @return The Number value */ public static Number cast( Object object, Boolean fail ) { + return cast( object, fail, false ); + } + + /** + * Used to cast anything to a Number + * + * @param object The value to cast to a Number + * @param fail If true, throw exception if we fail + * + * @return The Number value + */ + public static Number cast( Object object, Boolean fail, boolean castDates ) { if ( object == null ) { return 0; } @@ -110,6 +147,11 @@ public static Number cast( Object object, Boolean fail ) { } } + if ( castDates && DateTimeCaster.isKnownDateClass( object ) ) { + DateTime dObject = DateTimeCaster.cast( object ); + return dObject.toEpochMillis(); + } + // Try to parse the string as a Number String stringValue = StringCaster.cast( object, false ); Number result = parseNumber( stringValue ); diff --git a/src/main/java/ortus/boxlang/runtime/operators/Plus.java b/src/main/java/ortus/boxlang/runtime/operators/Plus.java index babf1cb5e..226eecd1b 100644 --- a/src/main/java/ortus/boxlang/runtime/operators/Plus.java +++ b/src/main/java/ortus/boxlang/runtime/operators/Plus.java @@ -43,8 +43,8 @@ public class Plus implements IOperator { * @return The the sum */ public static Number invoke( Object left, Object right ) { - Number nLeft = NumberCaster.cast( left ); - Number nRight = NumberCaster.cast( right ); + Number nLeft = NumberCaster.cast( true, left ); + Number nRight = NumberCaster.cast( true, right ); // A couple shortcuts-- if both operands are integers or longs within a certain range, we can just add them safely // If these checks turn into a performance overhead, we can remove them, but I was hoping it would be worth it since // BigDecimals are over twice the heap usage of a Double (~64 bits vs ~24 bits) diff --git a/src/test/java/TestCases/phase1/OperatorsTest.java b/src/test/java/TestCases/phase1/OperatorsTest.java index f8fb0b2f1..355ff6de5 100644 --- a/src/test/java/TestCases/phase1/OperatorsTest.java +++ b/src/test/java/TestCases/phase1/OperatorsTest.java @@ -983,4 +983,16 @@ public void testNotBeforeParens() { assertThat( result ).isEqualTo( false ); } + @DisplayName( "It can add numbers to a date" ) + @Test + public void testAddNumberToDate() { + instance.executeSource( """ + thisNow = now(); + thisNowMillis = thisNow.getTime(); + result = thisNow + 0; + """, context ); + assertThat( variables.get( resultKey ) ).isEqualTo( variables.get( Key.of( "thisNowMillis" ) ) ); + + } + } From ebd85cf5487eff14044737ce6ca814996e875dfc Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 9 Jan 2025 14:43:56 -0600 Subject: [PATCH 100/161] BL-910 --- .../expression/BoxAccessTransformer.java | 2 +- .../expression/BoxAccessTransformer.java | 2 +- src/test/java/TestCases/phase2/QueryTest.java | 21 ++++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java index 1952cb74b..3c953e136 100644 --- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxAccessTransformer.java @@ -137,7 +137,7 @@ public List transform( BoxNode node, TransformerContext contex // I don't know if this will work, but I'm trying to make an exception for query columns being passed to array BIFs // This prolly won't work if a query column is passed as a second param that isn't the array && ! ( parent instanceof BoxArgument barg && barg.getParent() instanceof BoxFunctionInvocation bfun - && bfun.getName().toLowerCase().contains( "array" ) ) ) { + && bfun.getName().toLowerCase().startsWith( "array" ) ) ) { nodes.addAll( 0, transpiler.getCurrentMethodContextTracker().get().loadCurrentContext() ); nodes.add( new MethodInsnNode( Opcodes.INVOKEINTERFACE, Type.getInternalName( IBoxContext.class ), diff --git a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAccessTransformer.java b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAccessTransformer.java index b80492a8f..bdb377114 100644 --- a/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAccessTransformer.java +++ b/src/main/java/ortus/boxlang/compiler/javaboxpiler/transformer/expression/BoxAccessTransformer.java @@ -103,7 +103,7 @@ public Node transform( BoxNode node, TransformerContext context ) throws Illegal // I don't know if this will work, but I'm trying to make an exception for query columns being passed to array BIFs // This prolly won't work if a query column is passed as a second param that isn't the array && ! ( parent instanceof BoxArgument barg && barg.getParent() instanceof BoxFunctionInvocation bfun - && bfun.getName().toLowerCase().contains( "array" ) ) ) { + && bfun.getName().toLowerCase().startsWith( "array" ) ) ) { template = "${contextName}.unwrapQueryColumn( " + template + " )"; } Node javaExpr = parseExpression( template, values ); diff --git a/src/test/java/TestCases/phase2/QueryTest.java b/src/test/java/TestCases/phase2/QueryTest.java index 14d86a6c0..4e2d1f9b0 100644 --- a/src/test/java/TestCases/phase2/QueryTest.java +++ b/src/test/java/TestCases/phase2/QueryTest.java @@ -33,6 +33,7 @@ import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.VariablesScope; +import ortus.boxlang.runtime.types.Array; import ortus.boxlang.runtime.types.IStruct; import ortus.boxlang.runtime.types.Query; import ortus.boxlang.runtime.types.exceptions.UnmodifiableException; @@ -110,7 +111,7 @@ public void testQuery() { @DisplayName( "Query Column assignment" ) @Disabled( "Issue with member function vs java method" ) @Test - public void testColumnAssignemtnQuery() { + public void testColumnAssignmentQuery() { // @formatter:off instance.executeSource( @@ -228,4 +229,22 @@ public void testUnmodifiableQueryErrors() { context ) ); } + @Test + public void testQueryColumnToArrayBIF() { + + instance.executeSource( + """ + myQry = queryNew( "col,col2", "numeric,varchar", [ [ 1, "brad,luis" ], [ 2, "" ], [ 3, "" ] ] ); + colAvg = arrayAvg( myQry.col ); + valList = listToArray( myQry.col2 ); + + """, + context ); + assertThat( variables.getAsNumber( Key.of( "colAvg" ) ).intValue() ).isEqualTo( 2 ); + assertThat( variables.get( Key.of( "valList" ) ) ).isInstanceOf( Array.class ); + assertThat( variables.getAsArray( Key.of( "valList" ) ).size() ).isEqualTo( 2 ); + assertThat( variables.getAsArray( Key.of( "valList" ) ).get( 0 ) ).isEqualTo( "brad" ); + assertThat( variables.getAsArray( Key.of( "valList" ) ).get( 1 ) ).isEqualTo( "luis" ); + } + } From 643c06184d39d405352dc9b0c3fd8c2ea762236e Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Thu, 9 Jan 2025 16:19:08 -0600 Subject: [PATCH 101/161] BL-907 --- .../runtime/components/system/Loop.java | 11 +++- .../runtime/components/system/LoopTest.java | 60 +++++++++++++++++-- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java index 5cf48852f..8b3533a46 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Loop.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Loop.java @@ -246,8 +246,17 @@ private BodyResult _invokeFile( IBoxContext context, String file, String index, private BodyResult _invokeRange( IBoxContext context, Double from, Double to, Number step, String index, ComponentBody body, IStruct executionState, String label ) { + double toD = to.doubleValue(); + double fromD = from.doubleValue(); + double stepD = step.doubleValue(); + // If step is positive, we loop until we're greater than or equal to the "to" value, otherwise we loop until we're less than or equal to the "to" value + java.util.function.Function condition = stepD > 0 ? i -> i <= toD : i -> i >= toD; + // Prevent infinite loops + if ( stepD == 0 ) { + return DEFAULT_RETURN; + } // Loop over array, executing body every time - for ( int i = from.intValue(); i <= to.intValue(); i = i + step.intValue() ) { + for ( double i = fromD; condition.apply( i ); i = i + stepD ) { // Set the index and item variables ExpressionInterpreter.setVariable( context, index, i ); // Run the code inside of the output loop diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java index 0124e54aa..ab3c2ef82 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java @@ -392,16 +392,68 @@ public void testLoopCondition() { instance.executeSource( """ function foo( required string name ) { - loop condition=arguments.name == "brad" { + loop condition=arguments.name == "brad" { return getFunctionCalledName(); - break; - } + break; + } } result = foo( "brad" ); - """, + """, context, BoxSourceType.BOXSCRIPT ); assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "foo" ); } + @Test + public void testLoopToFrom() { + instance.executeSource( + """ + result = "" + loop from="1" to="5" step="1" index="i" { + result &= i; + } + """, + context, BoxSourceType.BOXSCRIPT ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "12345" ); + } + + @Test + public void testLoopToFromNegativeStep() { + instance.executeSource( + """ + result = "" + loop from="5" to="1" step="-1" index="i" { + result &= i; + } + """, + context, BoxSourceType.BOXSCRIPT ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "54321" ); + } + + @Test + public void testLoopToFromZeroStep() { + instance.executeSource( + """ + result = "" + loop from="1" to="5" step="0" index="i" { + result &= i; + } + """, + context, BoxSourceType.BOXSCRIPT ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "" ); + } + + @Test + public void testLoopToFromDecimalStep() { + instance.executeSource( + """ + result = "" + loop from="1" to="10" step="1.5" index="i" { + result = result.listAppend(i) + } + """, + context, BoxSourceType.BOXSCRIPT ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "1,2.5,4,5.5,7,8.5,10" ); + } + } From fd79b448de33a3fdbdb85c2f74e1e37d9ac9b678 Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Thu, 9 Jan 2025 19:24:25 -0500 Subject: [PATCH 102/161] add tests for boolean false --- .../bifs/global/decision/IsNumericTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java index e8509933a..cf7e914ee 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java @@ -142,4 +142,21 @@ public void testADate() { assertFalse( variables.getAsBoolean( Key.of( "result" ) ) ); } + @DisplayName( "It will return false for a boolean value" ) + @Test + public void testBooleanFalse() { + instance.executeSource( + """ + result = isNumeric( true ) + """, + context ); + assertFalse( variables.getAsBoolean( Key.of( "result" ) ) ); + instance.executeSource( + """ + result = isNumeric( false ) + """, + context ); + assertFalse( variables.getAsBoolean( Key.of( "result" ) ) ); + } + } From c7459447651165d11abddd4556421ba751374c09 Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Thu, 9 Jan 2025 19:40:58 -0500 Subject: [PATCH 103/161] BL-911 resolve --- .../ortus/boxlang/runtime/bifs/global/decision/IsNumeric.java | 4 ++++ .../boxlang/runtime/bifs/global/decision/IsNumericTest.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/decision/IsNumeric.java b/src/main/java/ortus/boxlang/runtime/bifs/global/decision/IsNumeric.java index b142ad7fa..2aa826285 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/decision/IsNumeric.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/decision/IsNumeric.java @@ -73,6 +73,10 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { return false; } Locale locale = LocalizationUtil.parseLocaleFromContext( context, arguments ); + // We can't use the number caster on booleans when the booleansAreNumbers setting is set to true + if ( value instanceof Boolean ) { + return false; + } return GenericCaster.attempt( context, value, "numeric" ).wasSuccessful() ? true : StringCaster.attempt( value ).wasSuccessful() diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java index cf7e914ee..3f4e85e42 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/decision/IsNumericTest.java @@ -52,7 +52,6 @@ public static void setUp() { @AfterAll public static void teardown() { - } @BeforeEach From ae70755c0729c1cb46743e3436e2f7146745ae90 Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Fri, 10 Jan 2025 13:23:42 -0500 Subject: [PATCH 104/161] BL-913 resolve - fix throw with exception as first argument --- .../runtime/bifs/global/system/Throw.java | 8 ++++++-- .../runtime/components/system/Throw.java | 2 +- .../runtime/bifs/global/system/ThrowTest.java | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Throw.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Throw.java index 4c1e38ccb..54074633b 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Throw.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Throw.java @@ -35,7 +35,7 @@ public class Throw extends BIF { public Throw() { super(); declaredArguments = new Argument[] { - new Argument( false, "String", Key.message ), + new Argument( false, "any", Key.message ), new Argument( false, "String", Key.type ), new Argument( false, "String", Key.detail ), new Argument( false, "String", Key.errorcode ), @@ -64,7 +64,11 @@ public Throw() { * CustomException will be thrown and this object will be used as the cause. */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { - Throwable exceptionToThrow; + Throwable exceptionToThrow; + if ( arguments.get( Key.message ) instanceof Throwable ) { + arguments.put( Key.object, arguments.get( Key.message ) ); + arguments.put( Key.message, null ); + } String message = arguments.getAsString( Key.message ); String detail = arguments.getAsString( Key.detail ); String errorcode = arguments.getAsString( Key.errorcode ); diff --git a/src/main/java/ortus/boxlang/runtime/components/system/Throw.java b/src/main/java/ortus/boxlang/runtime/components/system/Throw.java index bbafbe5a1..16ef06b1d 100644 --- a/src/main/java/ortus/boxlang/runtime/components/system/Throw.java +++ b/src/main/java/ortus/boxlang/runtime/components/system/Throw.java @@ -36,7 +36,7 @@ public class Throw extends Component { public Throw() { super(); declaredAttributes = new Attribute[] { - new Attribute( Key.message, "String" ), + new Attribute( Key.message, "any" ), new Attribute( Key.type, "String" ), new Attribute( Key.detail, "String" ), new Attribute( Key.errorcode, "String" ), diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java index 31c8b80bd..c8f1849db 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ThrowTest.java @@ -187,4 +187,22 @@ public void testThrowJustType2() { assertThat( variables.get( Key.of( "type" ) ) ).isEqualTo( "DivideByZero" ); } + @Test + public void testThrowObjectUnnamed() { + //@formatter:off + instance.executeSource( """ + try { + throw( type="MyCustomException" ); + } catch ( e ) { + try{ + throw( e ); + } catch( e2 ) { + type = e2.type; + } + } + """, context, BoxSourceType.CFSCRIPT ); + //@formatter:on + assertThat( variables.get( Key.of( "type" ) ) ).isEqualTo( "MyCustomException" ); + } + } From 70190f17907cd98cb0aca2263fbcadef959e966e Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 10 Jan 2025 15:41:05 -0600 Subject: [PATCH 105/161] BL-912 --- .../ortus/boxlang/runtime/BoxRuntime.java | 34 ++-- .../application/ApplicationClassListener.java | 5 +- .../ApplicationDefaultListener.java | 5 +- .../ApplicationTemplateListener.java | 9 +- .../application/BaseApplicationListener.java | 17 +- .../global/system/GetBaseTemplatePath.java | 12 +- .../runtime/context/CatchBoxContext.java | 110 +----------- .../context/ConfigOverrideBoxContext.java | 63 +++++++ .../context/ParentPassthroughBoxContext.java | 169 ++++++++++++++++++ .../runtime/context/RequestBoxContext.java | 2 +- .../runtime/services/ApplicationService.java | 56 +++--- .../applications/ApplicationLookups.java | 166 +++++++++++++++++ .../applications/appClass/Application.bx | 8 + .../TestCases/applications/appClass/index.bxm | 1 + .../applications/appClass/sub1/index.bxm | 1 + .../applications/appClass/sub2/Application.bx | 8 + .../applications/appClass/sub2/include.bxm | 1 + .../applications/appClass/sub2/index.bxm | 3 + .../applications/appTemplate/Application.bxm | 1 + .../applications/appTemplate/index.bxm | 1 + .../applications/appTemplate/sub1/index.bxm | 1 + .../appTemplate/sub2/Application.bxm | 1 + .../applications/appTemplate/sub2/index.bxm | 1 + .../applications/external/Application.bx | 8 + .../TestCases/applications/external/index.bxm | 1 + .../applications/external/sub1/index.bxm | 1 + .../system/GetBaseTemplatePathTest.java | 89 +++------ 27 files changed, 548 insertions(+), 226 deletions(-) create mode 100644 src/main/java/ortus/boxlang/runtime/context/ConfigOverrideBoxContext.java create mode 100644 src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java create mode 100644 src/test/java/TestCases/applications/ApplicationLookups.java create mode 100644 src/test/java/TestCases/applications/appClass/Application.bx create mode 100644 src/test/java/TestCases/applications/appClass/index.bxm create mode 100644 src/test/java/TestCases/applications/appClass/sub1/index.bxm create mode 100644 src/test/java/TestCases/applications/appClass/sub2/Application.bx create mode 100644 src/test/java/TestCases/applications/appClass/sub2/include.bxm create mode 100644 src/test/java/TestCases/applications/appClass/sub2/index.bxm create mode 100644 src/test/java/TestCases/applications/appTemplate/Application.bxm create mode 100644 src/test/java/TestCases/applications/appTemplate/index.bxm create mode 100644 src/test/java/TestCases/applications/appTemplate/sub1/index.bxm create mode 100644 src/test/java/TestCases/applications/appTemplate/sub2/Application.bxm create mode 100644 src/test/java/TestCases/applications/appTemplate/sub2/index.bxm create mode 100644 src/test/java/TestCases/applications/external/Application.bx create mode 100644 src/test/java/TestCases/applications/external/index.bxm create mode 100644 src/test/java/TestCases/applications/external/sub1/index.bxm diff --git a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java index 50d5dbf4c..2dc80168e 100644 --- a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java +++ b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java @@ -1145,14 +1145,14 @@ public void executeTemplate( String templatePath, IBoxContext context, String[] // Load the class Class targetClass = RunnableLoader.getInstance().loadClass( ResolvedFilePath.of( Paths.get( templatePath ) ), - this.runtimeContext ); + context ); executeClass( targetClass, templatePath, context, args ); } else { // Load the template - BoxTemplate targetTemplate = RunnableLoader.getInstance().loadTemplateAbsolute( - this.runtimeContext, - ResolvedFilePath.of( Paths.get( templatePath ) ) ); - executeTemplate( targetTemplate, context ); + BoxTemplate targetTemplate = RunnableLoader.getInstance().loadTemplateRelative( + context, + templatePath ); + executeTemplate( targetTemplate, templatePath, context ); } } @@ -1312,14 +1312,28 @@ public void executeClass( Class targetClass, String templatePath, * @param context The context to execute the template in */ public void executeTemplate( BoxTemplate template, IBoxContext context ) { - String templatePath = template.getRunnablePath().absolutePath().toString(); + executeTemplate( template, template.getRunnablePath().absolutePath().toString(), context ); + } + + /** + * Execute a single template in an existing context using an already-loaded + * template runnable + * + * @param template A template to execute + * @param context The context to execute the template in + */ + public void executeTemplate( BoxTemplate template, String templatePath, IBoxContext context ) { instance.logger.debug( "Executing template [{}]", template.getRunnablePath() ); - IBoxContext scriptingContext = ensureRequestTypeContext( context, - template.getRunnablePath().absolutePath().toUri() ); - BaseApplicationListener listener = scriptingContext.getParentOfType( RequestBoxContext.class ) + IBoxContext scriptingContext; + try { + scriptingContext = ensureRequestTypeContext( context, new URI( templatePath ) ); + } catch ( URISyntaxException e ) { + throw new BoxRuntimeException( "Invalid template path to execute.", e ); + } + BaseApplicationListener listener = scriptingContext.getParentOfType( RequestBoxContext.class ) .getApplicationListener(); - Throwable errorToHandle = null; + Throwable errorToHandle = null; RequestBoxContext.setCurrent( scriptingContext.getParentOfType( RequestBoxContext.class ) ); ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); try { diff --git a/src/main/java/ortus/boxlang/runtime/application/ApplicationClassListener.java b/src/main/java/ortus/boxlang/runtime/application/ApplicationClassListener.java index c988ccdef..89b5b9112 100644 --- a/src/main/java/ortus/boxlang/runtime/application/ApplicationClassListener.java +++ b/src/main/java/ortus/boxlang/runtime/application/ApplicationClassListener.java @@ -31,6 +31,7 @@ import ortus.boxlang.runtime.types.util.BLCollector; import ortus.boxlang.runtime.util.EncryptionUtil; import ortus.boxlang.runtime.util.FileSystemUtil; +import ortus.boxlang.runtime.util.ResolvedFilePath; /** * I represent an Application listener that wraps an Application class instance, delegting to it, where possible and providing default @@ -49,8 +50,8 @@ public class ApplicationClassListener extends BaseApplicationListener { * @param listener An Application class instance * @param context The context to use */ - public ApplicationClassListener( IClassRunnable listener, RequestBoxContext context ) { - super( context ); + public ApplicationClassListener( IClassRunnable listener, RequestBoxContext context, ResolvedFilePath baseTemplatePath ) { + super( context, baseTemplatePath ); this.listener = listener; // Copy all the settings from the Application class to the settings map diff --git a/src/main/java/ortus/boxlang/runtime/application/ApplicationDefaultListener.java b/src/main/java/ortus/boxlang/runtime/application/ApplicationDefaultListener.java index ca8ce4d94..cf879615c 100644 --- a/src/main/java/ortus/boxlang/runtime/application/ApplicationDefaultListener.java +++ b/src/main/java/ortus/boxlang/runtime/application/ApplicationDefaultListener.java @@ -21,6 +21,7 @@ import ortus.boxlang.runtime.context.RequestBoxContext; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Struct; +import ortus.boxlang.runtime.util.ResolvedFilePath; /** * I represent a default Application listener @@ -32,8 +33,8 @@ public class ApplicationDefaultListener extends BaseApplicationListener { * * @param context The context to use */ - public ApplicationDefaultListener( RequestBoxContext context ) { - super( context ); + public ApplicationDefaultListener( RequestBoxContext context, ResolvedFilePath baseTemplatePath ) { + super( context, baseTemplatePath ); } /** diff --git a/src/main/java/ortus/boxlang/runtime/application/ApplicationTemplateListener.java b/src/main/java/ortus/boxlang/runtime/application/ApplicationTemplateListener.java index f554e6ea5..022c29022 100644 --- a/src/main/java/ortus/boxlang/runtime/application/ApplicationTemplateListener.java +++ b/src/main/java/ortus/boxlang/runtime/application/ApplicationTemplateListener.java @@ -22,6 +22,7 @@ import ortus.boxlang.runtime.runnables.BoxTemplate; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Struct; +import ortus.boxlang.runtime.util.ResolvedFilePath; /** * I represent an Application listener that wraps an Application template @@ -38,8 +39,8 @@ public class ApplicationTemplateListener extends BaseApplicationListener { * * @param listener An Application class instance */ - public ApplicationTemplateListener( BoxTemplate listener, RequestBoxContext context ) { - super( context ); + public ApplicationTemplateListener( BoxTemplate listener, RequestBoxContext context, ResolvedFilePath baseTemplatePath ) { + super( context, baseTemplatePath ); this.listener = listener; // Store the template path in the settings map this.settings.put( Key.source, listener.getRunnablePath().absolutePath().toString() ); @@ -55,8 +56,6 @@ public ApplicationTemplateListener( BoxTemplate listener, RequestBoxContext cont @Override public void onRequest( IBoxContext context, Object[] args ) { super.onRequest( context, args ); - // Run Application template - listener.invoke( context ); // Then include the requested template context.includeTemplate( ( String ) args[ 0 ] ); } @@ -64,6 +63,8 @@ public void onRequest( IBoxContext context, Object[] args ) { @Override public boolean onRequestStart( IBoxContext context, Object[] args ) { super.onRequestStart( context, args ); + // Run Application template + listener.invoke( context ); return true; } diff --git a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java index f63c141ca..0af7b5727 100644 --- a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java +++ b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java @@ -86,6 +86,14 @@ public abstract class BaseApplicationListener { */ protected InterceptorPool interceptorPool; + /** + * The template, if any, which initiated this request. + * For a web request, this is the URI + * For a scripting request, this is the file being executed + * Null for ad-hoc code execution. + */ + protected ResolvedFilePath baseTemplatePath = null; + /** * The available request pool interceptors */ @@ -169,8 +177,9 @@ public abstract class BaseApplicationListener { * * @param context The request context */ - protected BaseApplicationListener( RequestBoxContext context ) { - this.context = context; + protected BaseApplicationListener( RequestBoxContext context, ResolvedFilePath baseTemplatePath ) { + this.context = context; + this.baseTemplatePath = baseTemplatePath; context.setApplicationListener( this ); this.interceptorPool = new InterceptorPool( Key.appListener, BoxRuntime.getInstance() ) .registerInterceptionPoint( REQUEST_INTERCEPTION_POINTS ); @@ -937,4 +946,8 @@ public boolean onMissingTemplate( IBoxContext context, Object[] args ) { return true; } + + public ResolvedFilePath getBaseTemplatePath() { + return this.baseTemplatePath; + } } diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePath.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePath.java index 1d678d647..b176fbf83 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePath.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePath.java @@ -20,7 +20,9 @@ import ortus.boxlang.runtime.bifs.BIF; import ortus.boxlang.runtime.bifs.BoxBIF; import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.context.RequestBoxContext; import ortus.boxlang.runtime.scopes.ArgumentsScope; +import ortus.boxlang.runtime.util.ResolvedFilePath; @BoxBIF public class GetBaseTemplatePath extends BIF { @@ -40,6 +42,14 @@ public GetBaseTemplatePath() { * */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { - return context.findBaseTemplate().absolutePath().toString(); + RequestBoxContext requestContext = context.getRequestContext(); + if ( requestContext == null ) { + return ""; + } + ResolvedFilePath basepath = requestContext.getApplicationListener().getBaseTemplatePath(); + if ( basepath == null ) { + return ""; + } + return basepath.absolutePath().toString(); } } diff --git a/src/main/java/ortus/boxlang/runtime/context/CatchBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/CatchBoxContext.java index ece0fcdf1..450f156e8 100644 --- a/src/main/java/ortus/boxlang/runtime/context/CatchBoxContext.java +++ b/src/main/java/ortus/boxlang/runtime/context/CatchBoxContext.java @@ -18,9 +18,7 @@ package ortus.boxlang.runtime.context; import java.util.Map; -import java.util.function.Predicate; -import ortus.boxlang.runtime.components.Component; import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.ScopeWrapper; @@ -35,7 +33,7 @@ /** * This context represents the context of a template execution in BoxLang */ -public class CatchBoxContext extends BaseBoxContext { +public class CatchBoxContext extends ParentPassthroughBoxContext { /** * The variables scope @@ -196,16 +194,6 @@ public IScope getScopeNearby( Key name, boolean shallow ) { } - /** - * Get the default variable assignment scope for this context - * - * @return The scope reference to use - */ - public IScope getDefaultAssignmentScope() { - // parent is never null - return getParent().getDefaultAssignmentScope(); - } - /** * rethrows the closest exception */ @@ -213,100 +201,4 @@ public void rethrow() { ExceptionUtil.throwException( exception ); } - /** - * Most of these methods just delegate to the parent context so the catch context is mostly invisible. - */ - - public IBoxContext writeToBuffer( Object o ) { - if ( o == null ) { - return this; - } - getParent().writeToBuffer( o ); - return this; - } - - public IBoxContext writeToBuffer( Object o, boolean force ) { - if ( o == null ) { - return this; - } - getParent().writeToBuffer( o, force ); - return this; - } - - public Boolean canOutput() { - return getParent().canOutput(); - } - - public IBoxContext flushBuffer( boolean force ) { - getParent().flushBuffer( force ); - return this; - } - - public IBoxContext clearBuffer() { - getParent().clearBuffer(); - return this; - } - - public StringBuffer getBuffer() { - return getParent().getBuffer(); - } - - public IBoxContext pushBuffer( StringBuffer buffer ) { - getParent().pushBuffer( buffer ); - return this; - } - - public IBoxContext popBuffer() { - getParent().popBuffer(); - return this; - } - - public Object invokeFunction( Key name, Object[] positionalArguments ) { - return getParent().invokeFunction( name, positionalArguments ); - } - - public Object invokeFunction( Key name, Map namedArguments ) { - return getParent().invokeFunction( name, namedArguments ); - } - - public Object invokeFunction( Key name ) { - return getParent().invokeFunction( name ); - } - - public Object invokeFunction( Object function, Object[] positionalArguments ) { - return getParent().invokeFunction( function, positionalArguments ); - } - - public Object invokeFunction( Object function, Map namedArguments ) { - return getParent().invokeFunction( function, namedArguments ); - } - - public Object invokeFunction( Object function ) { - return getParent().invokeFunction( function ); - } - - public Component.BodyResult invokeComponent( Key name, IStruct attributes, Component.ComponentBody componentBody ) { - return getParent().invokeComponent( name, attributes, componentBody ); - } - - public IBoxContext pushComponent( IStruct executionState ) { - return getParent().pushComponent( executionState ); - } - - public IBoxContext popComponent() { - return getParent().popComponent(); - } - - public IStruct[] getComponents() { - return getParent().getComponents(); - } - - public IStruct findClosestComponent( Key name ) { - return getParent().findClosestComponent( name ); - } - - public IStruct findClosestComponent( Key name, Predicate predicate ) { - return getParent().findClosestComponent( name, predicate ); - } - } diff --git a/src/main/java/ortus/boxlang/runtime/context/ConfigOverrideBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/ConfigOverrideBoxContext.java new file mode 100644 index 000000000..9bdb06628 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/context/ConfigOverrideBoxContext.java @@ -0,0 +1,63 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ortus.boxlang.runtime.context; + +import java.util.function.Function; + +import ortus.boxlang.runtime.types.IStruct; + +/** + * This context provides a way to override config for all downstream execution. + */ +public class ConfigOverrideBoxContext extends ParentPassthroughBoxContext { + + /** + * -------------------------------------------------------------------------- + * Private Properties + * -------------------------------------------------------------------------- + */ + + /** + * The variables scope + */ + protected Function configOverride; + + /** + * -------------------------------------------------------------------------- + * Constructors + * -------------------------------------------------------------------------- + */ + + /** + * Creates a new execution context with a bounded execution template and parent context + * + * @param parent The parent context + */ + public ConfigOverrideBoxContext( IBoxContext parent, Function configOverride ) { + super( parent ); + this.configOverride = configOverride; + } + + /** + * Allow our overrides to happen + */ + public IStruct getConfig() { + var config = super.getConfig(); + return configOverride.apply( config ); + } +} diff --git a/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java new file mode 100644 index 000000000..4a5afc179 --- /dev/null +++ b/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java @@ -0,0 +1,169 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ortus.boxlang.runtime.context; + +import java.util.Map; +import java.util.function.Predicate; + +import ortus.boxlang.runtime.components.Component; +import ortus.boxlang.runtime.scopes.IScope; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.types.IStruct; + +/** + * This context provides a base class for contexts which add some functionality, but mostly pass through all of their methods + * to their parent context. + */ +public abstract class ParentPassthroughBoxContext extends BaseBoxContext { + + /** + * -------------------------------------------------------------------------- + * Constructors + * -------------------------------------------------------------------------- + */ + + /** + * Creates a new execution context with a bounded execution template and parent context + * + * @param parent The parent context + */ + public ParentPassthroughBoxContext( IBoxContext parent ) { + super( parent ); + } + + /** + * All these methods just delegate to the parent context so the config override context is mostly invisible. + */ + + public IStruct getVisibleScopes( IStruct scopes, boolean nearby, boolean shallow ) { + return getParent().getVisibleScopes( scopes, nearby, shallow ); + } + + public ScopeSearchResult scopeFindNearby( Key key, IScope defaultScope, boolean shallow, boolean forAssign ) { + return getParent().scopeFindNearby( key, defaultScope, shallow, forAssign ); + } + + public ScopeSearchResult scopeFind( Key key, IScope defaultScope, boolean forAssign ) { + return getParent().scopeFind( key, defaultScope, forAssign ); + } + + public IScope getScope( Key name ) { + return getParent().getScope( name ); + } + + public IScope getScopeNearby( Key name, boolean shallow ) { + return getParent().getScopeNearby( name, shallow ); + } + + public IScope getDefaultAssignmentScope() { + return getParent().getDefaultAssignmentScope(); + } + + public IBoxContext writeToBuffer( Object o ) { + if ( o == null ) { + return this; + } + getParent().writeToBuffer( o ); + return this; + } + + public IBoxContext writeToBuffer( Object o, boolean force ) { + if ( o == null ) { + return this; + } + getParent().writeToBuffer( o, force ); + return this; + } + + public Boolean canOutput() { + return getParent().canOutput(); + } + + public IBoxContext flushBuffer( boolean force ) { + getParent().flushBuffer( force ); + return this; + } + + public IBoxContext clearBuffer() { + getParent().clearBuffer(); + return this; + } + + public StringBuffer getBuffer() { + return getParent().getBuffer(); + } + + public IBoxContext pushBuffer( StringBuffer buffer ) { + getParent().pushBuffer( buffer ); + return this; + } + + public IBoxContext popBuffer() { + getParent().popBuffer(); + return this; + } + + public Object invokeFunction( Key name, Object[] positionalArguments ) { + return getParent().invokeFunction( name, positionalArguments ); + } + + public Object invokeFunction( Key name, Map namedArguments ) { + return getParent().invokeFunction( name, namedArguments ); + } + + public Object invokeFunction( Key name ) { + return getParent().invokeFunction( name ); + } + + public Object invokeFunction( Object function, Object[] positionalArguments ) { + return getParent().invokeFunction( function, positionalArguments ); + } + + public Object invokeFunction( Object function, Map namedArguments ) { + return getParent().invokeFunction( function, namedArguments ); + } + + public Object invokeFunction( Object function ) { + return getParent().invokeFunction( function ); + } + + public Component.BodyResult invokeComponent( Key name, IStruct attributes, Component.ComponentBody componentBody ) { + return getParent().invokeComponent( name, attributes, componentBody ); + } + + public IBoxContext pushComponent( IStruct executionState ) { + return getParent().pushComponent( executionState ); + } + + public IBoxContext popComponent() { + return getParent().popComponent(); + } + + public IStruct[] getComponents() { + return getParent().getComponents(); + } + + public IStruct findClosestComponent( Key name ) { + return getParent().findClosestComponent( name ); + } + + public IStruct findClosestComponent( Key name, Predicate predicate ) { + return getParent().findClosestComponent( name, predicate ); + } + +} diff --git a/src/main/java/ortus/boxlang/runtime/context/RequestBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/RequestBoxContext.java index fe3cb2964..cb92ad683 100644 --- a/src/main/java/ortus/boxlang/runtime/context/RequestBoxContext.java +++ b/src/main/java/ortus/boxlang/runtime/context/RequestBoxContext.java @@ -243,7 +243,7 @@ public BaseApplicationListener getApplicationListener() { // Since we've hit a code path that requires the applicationListener, we'll create it if it doesn't exist // using our default one. It will likely get replaced, but for now can provide default values. if ( this.applicationListener == null ) { - this.applicationListener = new ApplicationDefaultListener( this ); + this.applicationListener = new ApplicationDefaultListener( this, null ); } return this.applicationListener; } diff --git a/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java b/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java index 9f3983089..31cd48858 100644 --- a/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java +++ b/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java @@ -40,6 +40,8 @@ import ortus.boxlang.runtime.runnables.RunnableLoader; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.Struct; +import ortus.boxlang.runtime.util.BoxFQN; +import ortus.boxlang.runtime.util.FileSystemUtil; import ortus.boxlang.runtime.util.ResolvedFilePath; /** @@ -217,38 +219,37 @@ public void onShutdown( Boolean force ) { public BaseApplicationListener createApplicationListener( RequestBoxContext context, URI template ) { BaseApplicationListener listener; ApplicationDescriptorSearch searchResult = null; + ResolvedFilePath templatePath = null; if ( template != null ) { // Look for an Application descriptor based on our lookup rules String directoryOfTemplate = null; String packagePath = ""; - String rootMapping = context.getConfig().getAsStruct( Key.mappings ) - .getAsString( Key._slash ); if ( template.isAbsolute() ) { + templatePath = ResolvedFilePath.of( template.getPath() ); directoryOfTemplate = new File( template ).getParent(); searchResult = fileLookup( directoryOfTemplate ); } else { - directoryOfTemplate = new File( template.toString() ).getParent(); - while ( directoryOfTemplate != null ) { - if ( directoryOfTemplate.equals( File.separator ) ) { - searchResult = fileLookup( rootMapping ); - } else { - searchResult = fileLookup( Paths.get( rootMapping, directoryOfTemplate ).toString() ); - } + // This may not be the actual absolute path of the file if we're including a file which is being + // resolved via a mapping declared in the Application class, which we haven't yet created + + templatePath = FileSystemUtil.expandPath( context, template.getPath().toString() ); + Path rootPath = Paths.get( templatePath.mappingPath() ); + Path currentDirectory = templatePath.absolutePath().getParent(); + while ( currentDirectory != null && ( currentDirectory.startsWith( rootPath ) || currentDirectory.equals( rootPath ) ) ) { + searchResult = fileLookup( currentDirectory.toString() ); if ( searchResult != null ) { - // set packagePath to the relative path from the rootMapping to the - // directoryOfTemplate with slashes replaced with dots - packagePath = directoryOfTemplate.replace( File.separator, "." ); - if ( packagePath.endsWith( "." ) ) { - packagePath = packagePath.substring( 0, packagePath.length() - 1 ); - } - // trim leading . - if ( packagePath.startsWith( "." ) ) { - packagePath = packagePath.substring( 1 ); + // Combine the mapping name with the relative path still left to the template + String mappingDotPath = templatePath.mappingName().equals( "/" ) ? "" + : templatePath.mappingName().substring( 1 ).replace( File.separator, "." ); + if ( !mappingDotPath.isBlank() && !mappingDotPath.endsWith( "." ) ) { + mappingDotPath += "."; } + packagePath = new BoxFQN( + mappingDotPath + currentDirectory.toString().substring( rootPath.toString().length() ).replace( File.separator, "." ) ).toString(); break; } - directoryOfTemplate = new File( directoryOfTemplate ).getParent(); + currentDirectory = currentDirectory.getParent(); } } // If we found an Application class, instantiate it @@ -259,8 +260,8 @@ public BaseApplicationListener createApplicationListener( RequestBoxContext cont RunnableLoader.getInstance() .loadClass( ResolvedFilePath.of( - "/", - rootMapping, + templatePath.mappingName(), + templatePath.mappingPath(), packagePath.replace( ".", File.separator ) + File.separator + searchResult.path().getFileName(), searchResult.path() ), @@ -268,28 +269,29 @@ public BaseApplicationListener createApplicationListener( RequestBoxContext cont // We do NOT invoke init() on the Application class for CF compat .invokeConstructor( context, Key.noInit ) .getTargetInstance(), - context ); + context, + templatePath ); } else { // If we found a template, return a new empty ApplicationListener listener = new ApplicationTemplateListener( RunnableLoader.getInstance().loadTemplateAbsolute( context, ResolvedFilePath.of( - "/", - rootMapping, + templatePath.mappingName(), + templatePath.mappingPath(), packagePath.replace( ".", File.separator ) + File.separator + searchResult.path().getFileName(), searchResult.path() ) ), - context ); + context, templatePath ); } } else { // If we didn't find an Application, return a new empty ApplicationListener - listener = new ApplicationDefaultListener( context ); + listener = new ApplicationDefaultListener( context, templatePath ); } } else { // If we didn't have a template, return a new empty ApplicationListener - listener = new ApplicationDefaultListener( context ); + listener = new ApplicationDefaultListener( context, templatePath ); } // Announce event so modules can hook in diff --git a/src/test/java/TestCases/applications/ApplicationLookups.java b/src/test/java/TestCases/applications/ApplicationLookups.java new file mode 100644 index 000000000..a78eb23e7 --- /dev/null +++ b/src/test/java/TestCases/applications/ApplicationLookups.java @@ -0,0 +1,166 @@ +/** + * [BoxLang] + * + * Copyright [2023] [Ortus Solutions, Corp] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package TestCases.applications; + +import static com.google.common.truth.Truth.assertThat; + +import java.net.URI; +import java.net.URISyntaxException; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import ortus.boxlang.runtime.BoxRuntime; +import ortus.boxlang.runtime.context.ConfigOverrideBoxContext; +import ortus.boxlang.runtime.context.IBoxContext; +import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; +import ortus.boxlang.runtime.scopes.IScope; +import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.scopes.RequestScope; + +public class ApplicationLookups { + + static BoxRuntime instance; + IBoxContext context; + static Key result = new Key( "result" ); + + @BeforeAll + public static void setUp() { + instance = BoxRuntime.getInstance( true ); + } + + @BeforeEach + public void setupEach() { + } + + @Test + public void testAppTemplateInRoot() { + context = getContext( "src/test/java/TestCases/applications/appTemplate/", "index.bxm" ); + instance.executeTemplate( + "index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxmran" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmran" ) ).isEqualTo( true ); + } + + @Test + public void testAppClassInRoot() { + context = getContext( "src/test/java/TestCases/applications/appClass", "index.bxm" ); + instance.executeTemplate( + "index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxran" ) ).isEqualTo( true ); + assertThat( request.get( "onRequestStart" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmran" ) ).isEqualTo( true ); + } + + @Test + public void testAppTemplateInEmptySub() { + context = getContext( "src/test/java/TestCases/applications/appTemplate/", "sub1/index.bxm" ); + instance.executeTemplate( + "sub1/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxmran" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmsub1ran" ) ).isEqualTo( true ); + } + + @Test + public void testAppClassInAppSub() { + context = getContext( "src/test/java/TestCases/applications/appClass/", "sub2/index.bxm" ); + instance.executeTemplate( + "sub2/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxsub2ran" ) ).isEqualTo( true ); + assertThat( request.get( "onRequestStartsub2" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmsub2ran" ) ).isEqualTo( true ); + } + + @Test + public void testAppTemplateInAppSub() { + context = getContext( "src/test/java/TestCases/applications/appTemplate/", "sub2/index.bxm" ); + instance.executeTemplate( + "sub2/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxmsub2ran" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmsub2ran" ) ).isEqualTo( true ); + } + + @Test + public void testAppClassInEmptySub() { + context = getContext( "src/test/java/TestCases/applications/appClass/", "sub2/index.bxm" ); + instance.executeTemplate( + "sub2/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "applicationbxsub2ran" ) ).isEqualTo( true ); + assertThat( request.get( "onRequestStartsub2" ) ).isEqualTo( true ); + assertThat( request.get( "indexbxmsub2ran" ) ).isEqualTo( true ); + } + + @Test + public void testAppClassInMapping() { + instance.getConfiguration().mappings.put( "/secret", new java.io.File( "src/test/java/TestCases/applications/external" ).getAbsolutePath() ); + context = getContext( "src/test/java/TestCases/applications/appClass/", "secret/index.bxm" ); + instance.executeTemplate( + "secret/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "externalapplicationbxran" ) ).isEqualTo( true ); + assertThat( request.get( "externalonRequestStart" ) ).isEqualTo( true ); + assertThat( request.get( "externalindexbxmran" ) ).isEqualTo( true ); + } + + @Test + public void testAppClassInMappingSub() { + instance.getConfiguration().mappings.put( "/secret", new java.io.File( "src/test/java/TestCases/applications/external" ).getAbsolutePath() ); + context = getContext( "src/test/java/TestCases/applications/appClass/", "secret/sub1/index.bxm" ); + instance.executeTemplate( + "secret/sub1/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + assertThat( request.get( "externalapplicationbxran" ) ).isEqualTo( true ); + assertThat( request.get( "externalonRequestStart" ) ).isEqualTo( true ); + assertThat( request.get( "externalindexbxmsub1ran" ) ).isEqualTo( true ); + } + + private IBoxContext getContext( String rootPath, String template ) { + try { + return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { + config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); + return config; + } ), new URI( template ) ); + } catch ( URISyntaxException e ) { + throw new RuntimeException( e ); + } + } + +} diff --git a/src/test/java/TestCases/applications/appClass/Application.bx b/src/test/java/TestCases/applications/appClass/Application.bx new file mode 100644 index 000000000..e64395e90 --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/Application.bx @@ -0,0 +1,8 @@ +class { + + request.applicationbxran = true; + + function onRequestStart() { + request.onRequestStart = true; + } +} diff --git a/src/test/java/TestCases/applications/appClass/index.bxm b/src/test/java/TestCases/applications/appClass/index.bxm new file mode 100644 index 000000000..2f35bd1b2 --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appClass/sub1/index.bxm b/src/test/java/TestCases/applications/appClass/sub1/index.bxm new file mode 100644 index 000000000..22d0b7ab8 --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/sub1/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appClass/sub2/Application.bx b/src/test/java/TestCases/applications/appClass/sub2/Application.bx new file mode 100644 index 000000000..ed578d5eb --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/sub2/Application.bx @@ -0,0 +1,8 @@ +class { + + request.applicationbxsub2ran = true; + + function onRequestStart() { + request.onRequestStartsub2 = true; + } +} diff --git a/src/test/java/TestCases/applications/appClass/sub2/include.bxm b/src/test/java/TestCases/applications/appClass/sub2/include.bxm new file mode 100644 index 000000000..10f5124f0 --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/sub2/include.bxm @@ -0,0 +1 @@ + diff --git a/src/test/java/TestCases/applications/appClass/sub2/index.bxm b/src/test/java/TestCases/applications/appClass/sub2/index.bxm new file mode 100644 index 000000000..3bea76294 --- /dev/null +++ b/src/test/java/TestCases/applications/appClass/sub2/index.bxm @@ -0,0 +1,3 @@ + + + diff --git a/src/test/java/TestCases/applications/appTemplate/Application.bxm b/src/test/java/TestCases/applications/appTemplate/Application.bxm new file mode 100644 index 000000000..2c16dc22c --- /dev/null +++ b/src/test/java/TestCases/applications/appTemplate/Application.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appTemplate/index.bxm b/src/test/java/TestCases/applications/appTemplate/index.bxm new file mode 100644 index 000000000..2f35bd1b2 --- /dev/null +++ b/src/test/java/TestCases/applications/appTemplate/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appTemplate/sub1/index.bxm b/src/test/java/TestCases/applications/appTemplate/sub1/index.bxm new file mode 100644 index 000000000..22d0b7ab8 --- /dev/null +++ b/src/test/java/TestCases/applications/appTemplate/sub1/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appTemplate/sub2/Application.bxm b/src/test/java/TestCases/applications/appTemplate/sub2/Application.bxm new file mode 100644 index 000000000..54cb844bb --- /dev/null +++ b/src/test/java/TestCases/applications/appTemplate/sub2/Application.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/appTemplate/sub2/index.bxm b/src/test/java/TestCases/applications/appTemplate/sub2/index.bxm new file mode 100644 index 000000000..c0259c580 --- /dev/null +++ b/src/test/java/TestCases/applications/appTemplate/sub2/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/external/Application.bx b/src/test/java/TestCases/applications/external/Application.bx new file mode 100644 index 000000000..4878e1b94 --- /dev/null +++ b/src/test/java/TestCases/applications/external/Application.bx @@ -0,0 +1,8 @@ +class { + + request.externalapplicationbxran = true; + + function onRequestStart() { + request.externalonRequestStart = true; + } +} diff --git a/src/test/java/TestCases/applications/external/index.bxm b/src/test/java/TestCases/applications/external/index.bxm new file mode 100644 index 000000000..aa46a76cb --- /dev/null +++ b/src/test/java/TestCases/applications/external/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/TestCases/applications/external/sub1/index.bxm b/src/test/java/TestCases/applications/external/sub1/index.bxm new file mode 100644 index 000000000..d764ac93a --- /dev/null +++ b/src/test/java/TestCases/applications/external/sub1/index.bxm @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java index 91222ca5e..ce77c6f4f 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java @@ -20,9 +20,8 @@ import static com.google.common.truth.Truth.assertThat; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.List; +import java.net.URI; +import java.net.URISyntaxException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -30,16 +29,14 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import ortus.boxlang.compiler.parser.BoxSourceType; import ortus.boxlang.runtime.BoxRuntime; +import ortus.boxlang.runtime.context.ConfigOverrideBoxContext; import ortus.boxlang.runtime.context.IBoxContext; import ortus.boxlang.runtime.context.ScriptingRequestBoxContext; -import ortus.boxlang.runtime.loader.ImportDefinition; -import ortus.boxlang.runtime.runnables.BoxTemplate; import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; +import ortus.boxlang.runtime.scopes.RequestScope; import ortus.boxlang.runtime.scopes.VariablesScope; -import ortus.boxlang.runtime.util.ResolvedFilePath; public class GetBaseTemplatePathTest { @@ -67,72 +64,28 @@ public void setupEach() { @DisplayName( "It gets base template path" ) @Test public void testBaseTemplate() { - context.pushTemplate( new BoxTemplate() { - - @Override - public List getImports() { - return null; - } - - @Override - public void _invoke( IBoxContext context ) { - } - - @Override - public long getRunnableCompileVersion() { - return 1; - } - - @Override - public LocalDateTime getRunnableCompiledOn() { - return null; - } - - @Override - public Object getRunnableAST() { - return null; - } - - @Override - public ResolvedFilePath getRunnablePath() { - return ResolvedFilePath.of( Path.of( "/tmp/test.bxs" ) ); - } - - public BoxSourceType getSourceType() { - return BoxSourceType.BOXSCRIPT; - } - - } ); - - instance.executeSource( - """ - result = getBaseTemplatePath(); - """, + + context = getContext( "src/test/java/TestCases/applications/appClass/", "sub2/index.bxm" ); + instance.executeTemplate( + "sub2/index.bxm", context ); - assertThat( variables.get( result ).toString().contains( "test.bxs" ) ).isTrue(); - instance.executeSource( - """ - // TODO: Move to compat module if/when CFTranspilerVisitor moves there - result = getTemplatePath(); - """, - context, BoxSourceType.CFSCRIPT ); - assertThat( variables.get( result ).toString().contains( "test.bxs" ) ).isTrue(); + IScope request = context.getScopeNearby( RequestScope.name ); + String actualPath = new java.io.File( "src/test/java/TestCases/applications/appClass/sub2/index.bxm" ).getAbsolutePath(); + assertThat( request.get( "indexbxmBasePath" ) ).isEqualTo( actualPath ); + assertThat( request.get( "includexmBasePath" ) ).isEqualTo( actualPath ); - context.popTemplate(); } - @DisplayName( "It gets base template path in include" ) - @Test - public void testBaseTemplateInclude() { - - instance.executeSource( - """ - include "src/test/java/ortus/boxlang/runtime/bifs/global/system/BaseTest1.cfs"; - """, - context ); - assertThat( variables.get( result ) ).isInstanceOf( String.class ); - assertThat( variables.getAsString( result ).contains( "BaseTest3.cfs" ) ).isTrue(); + private IBoxContext getContext( String rootPath, String template ) { + try { + return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { + config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); + return config; + } ), new URI( template ) ); + } catch ( URISyntaxException e ) { + throw new RuntimeException( e ); + } } } From 8d366907421224ab033dc9a136eacc6d1811dee1 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 10 Jan 2025 15:43:59 -0600 Subject: [PATCH 106/161] BL-912 --- .../applications/external/sub1/include.bxm | 1 + .../TestCases/applications/external/sub1/index.bxm | 4 +++- .../global/system/GetBaseTemplatePathTest.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/test/java/TestCases/applications/external/sub1/include.bxm diff --git a/src/test/java/TestCases/applications/external/sub1/include.bxm b/src/test/java/TestCases/applications/external/sub1/include.bxm new file mode 100644 index 000000000..57f2e67ee --- /dev/null +++ b/src/test/java/TestCases/applications/external/sub1/include.bxm @@ -0,0 +1 @@ + diff --git a/src/test/java/TestCases/applications/external/sub1/index.bxm b/src/test/java/TestCases/applications/external/sub1/index.bxm index d764ac93a..e650cb516 100644 --- a/src/test/java/TestCases/applications/external/sub1/index.bxm +++ b/src/test/java/TestCases/applications/external/sub1/index.bxm @@ -1 +1,3 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java index ce77c6f4f..03e113558 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java @@ -77,6 +77,20 @@ public void testBaseTemplate() { } + @Test + public void testAppClassInMappingSub() { + instance.getConfiguration().mappings.put( "/secret", new java.io.File( "src/test/java/TestCases/applications/external" ).getAbsolutePath() ); + context = getContext( "src/test/java/TestCases/applications/appClass/", "secret/sub1/index.bxm" ); + instance.executeTemplate( + "secret/sub1/index.bxm", + context ); + + IScope request = context.getScopeNearby( RequestScope.name ); + String actualPath = new java.io.File( "src/test/java/TestCases/applications/external/sub1/index.bxm" ).getAbsolutePath(); + assertThat( request.get( "externalindexbxmBasePath" ) ).isEqualTo( actualPath ); + assertThat( request.get( "externalincludexmBasePath" ) ).isEqualTo( actualPath ); + } + private IBoxContext getContext( String rootPath, String template ) { try { return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { From 453e0db8e5a3550af10c2d7338ac18853a56182f Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 10 Jan 2025 17:13:02 -0600 Subject: [PATCH 107/161] remove output --- .../boxlang/runtime/bifs/global/system/GetFunctionListTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetFunctionListTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetFunctionListTest.java index bfa9996c8..d106fcd4b 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetFunctionListTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetFunctionListTest.java @@ -61,7 +61,6 @@ public void testGetFunctionList() { context ); // @formatter:on - System.out.println( variables.get( result ) ); IStruct functions = ( IStruct ) variables.get( result ); assertThat( functions ).isNotNull(); From 30c71a835f697af9f475e90964223dbb1e05da85 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 10 Jan 2025 21:35:14 -0600 Subject: [PATCH 108/161] BL-907 --- .../ortus/boxlang/runtime/BoxRuntime.java | 7 ++--- .../runtime/services/ApplicationService.java | 5 ++- .../boxlang/runtime/util/FileSystemUtil.java | 31 +++++++++++++++++++ .../applications/ApplicationLookups.java | 16 +++------- .../system/GetBaseTemplatePathTest.java | 17 +++------- .../global/system/SessionInvalidateTest.java | 12 ++----- 6 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java index 2dc80168e..e8195907e 100644 --- a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java +++ b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java @@ -88,6 +88,7 @@ import ortus.boxlang.runtime.types.exceptions.MissingIncludeException; import ortus.boxlang.runtime.types.util.MathUtil; import ortus.boxlang.runtime.util.EncryptionUtil; +import ortus.boxlang.runtime.util.FileSystemUtil; import ortus.boxlang.runtime.util.ResolvedFilePath; import ortus.boxlang.runtime.util.Timer; @@ -1326,11 +1327,7 @@ public void executeTemplate( BoxTemplate template, String templatePath, IBoxCont instance.logger.debug( "Executing template [{}]", template.getRunnablePath() ); IBoxContext scriptingContext; - try { - scriptingContext = ensureRequestTypeContext( context, new URI( templatePath ) ); - } catch ( URISyntaxException e ) { - throw new BoxRuntimeException( "Invalid template path to execute.", e ); - } + scriptingContext = ensureRequestTypeContext( context, FileSystemUtil.createFileUri( templatePath ) ); BaseApplicationListener listener = scriptingContext.getParentOfType( RequestBoxContext.class ) .getApplicationListener(); Throwable errorToHandle = null; diff --git a/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java b/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java index 31cd48858..c7b362099 100644 --- a/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java +++ b/src/main/java/ortus/boxlang/runtime/services/ApplicationService.java @@ -221,19 +221,18 @@ public BaseApplicationListener createApplicationListener( RequestBoxContext cont ApplicationDescriptorSearch searchResult = null; ResolvedFilePath templatePath = null; if ( template != null ) { - // Look for an Application descriptor based on our lookup rules String directoryOfTemplate = null; String packagePath = ""; if ( template.isAbsolute() ) { - templatePath = ResolvedFilePath.of( template.getPath() ); + templatePath = ResolvedFilePath.of( Paths.get( template ) ); directoryOfTemplate = new File( template ).getParent(); searchResult = fileLookup( directoryOfTemplate ); } else { // This may not be the actual absolute path of the file if we're including a file which is being // resolved via a mapping declared in the Application class, which we haven't yet created - templatePath = FileSystemUtil.expandPath( context, template.getPath().toString() ); + templatePath = FileSystemUtil.expandPath( context, template.getPath() ); Path rootPath = Paths.get( templatePath.mappingPath() ); Path currentDirectory = templatePath.absolutePath().getParent(); while ( currentDirectory != null && ( currentDirectory.startsWith( rootPath ) || currentDirectory.equals( rootPath ) ) ) { diff --git a/src/main/java/ortus/boxlang/runtime/util/FileSystemUtil.java b/src/main/java/ortus/boxlang/runtime/util/FileSystemUtil.java index e2b62a854..ff98992b7 100644 --- a/src/main/java/ortus/boxlang/runtime/util/FileSystemUtil.java +++ b/src/main/java/ortus/boxlang/runtime/util/FileSystemUtil.java @@ -1187,4 +1187,35 @@ private static boolean caseSensitivityCheck() { return true; } + /** + * Creates a URI from a file path string + * + * @param input the file path string + * + * @return the URI + * + * @throws Exception + */ + public static URI createFileUri( String input ) { + try { + if ( input.startsWith( "/" ) || input.contains( ":" ) ) { + // Absolute path: ensure "file://" scheme starts with "file:///" + // Convert backslashes to forward slashes and prepend "file:///" + if ( input.matches( "^[A-Za-z]:.*" ) ) { + // Windows absolute path (e.g., C:\path\to\file) + input = "file:///" + input.replace( "\\", "/" ); + } else { + // Unix-style absolute path (e.g., /path/to/file) + input = "file://" + input.replace( "\\", "/" ); + } + return new URI( input ); + } else { + // Relative path: just replace backslashes with forward slashes + return new URI( input.replace( "\\", "/" ) ); + } + } catch ( URISyntaxException e ) { + throw new RuntimeException( "The provided file path [" + input + "] is not a valid URI.", e ); + } + } + } diff --git a/src/test/java/TestCases/applications/ApplicationLookups.java b/src/test/java/TestCases/applications/ApplicationLookups.java index a78eb23e7..3956370f3 100644 --- a/src/test/java/TestCases/applications/ApplicationLookups.java +++ b/src/test/java/TestCases/applications/ApplicationLookups.java @@ -19,9 +19,6 @@ import static com.google.common.truth.Truth.assertThat; -import java.net.URI; -import java.net.URISyntaxException; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,6 +30,7 @@ import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.RequestScope; +import ortus.boxlang.runtime.util.FileSystemUtil; public class ApplicationLookups { @@ -153,14 +151,10 @@ public void testAppClassInMappingSub() { } private IBoxContext getContext( String rootPath, String template ) { - try { - return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { - config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); - return config; - } ), new URI( template ) ); - } catch ( URISyntaxException e ) { - throw new RuntimeException( e ); - } + return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { + config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); + return config; + } ), FileSystemUtil.createFileUri( template ) ); } } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java index 03e113558..9546de85e 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/GetBaseTemplatePathTest.java @@ -20,9 +20,6 @@ import static com.google.common.truth.Truth.assertThat; -import java.net.URI; -import java.net.URISyntaxException; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -37,6 +34,7 @@ import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.RequestScope; import ortus.boxlang.runtime.scopes.VariablesScope; +import ortus.boxlang.runtime.util.FileSystemUtil; public class GetBaseTemplatePathTest { @@ -92,14 +90,9 @@ public void testAppClassInMappingSub() { } private IBoxContext getContext( String rootPath, String template ) { - try { - return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { - config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); - return config; - } ), new URI( template ) ); - } catch ( URISyntaxException e ) { - throw new RuntimeException( e ); - } + return new ScriptingRequestBoxContext( new ConfigOverrideBoxContext( instance.getRuntimeContext(), config -> { + config.getAsStruct( Key.mappings ).put( "/", new java.io.File( rootPath ).getAbsolutePath() ); + return config; + } ), FileSystemUtil.createFileUri( template ) ); } - } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java index e1a1f27e7..cb1ccb50a 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java @@ -22,9 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import java.net.URI; -import java.net.URISyntaxException; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -38,6 +35,7 @@ import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.VariablesScope; import ortus.boxlang.runtime.types.IStruct; +import ortus.boxlang.runtime.util.FileSystemUtil; public class SessionInvalidateTest { @@ -83,12 +81,8 @@ public void testBif() { @DisplayName( "It tests onSessionEnd" ) @Test public void testOnSessionEnd() { - try { - context = new ScriptingRequestBoxContext( instance.getRuntimeContext(), - new URI( "src/test/java/ortus/boxlang/runtime/bifs/global/system/testApp/index.bxm" ) ); - } catch ( URISyntaxException e ) { - throw new RuntimeException( e ); - } + context = new ScriptingRequestBoxContext( instance.getRuntimeContext(), + FileSystemUtil.createFileUri( "src/test/java/ortus/boxlang/runtime/bifs/global/system/testApp/index.bxm" ) ); instance.executeSource( """ application.brad = "wood"; From 244b5579bdb6501c1682fe8c5634726a227463d3 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Fri, 10 Jan 2025 23:38:56 -0600 Subject: [PATCH 109/161] BL-906 --- .../context/ParentPassthroughBoxContext.java | 5 ----- .../java/TestCases/components/BoxTemplateTest.java | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java index 4a5afc179..6befdaa35 100644 --- a/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java +++ b/src/main/java/ortus/boxlang/runtime/context/ParentPassthroughBoxContext.java @@ -20,7 +20,6 @@ import java.util.Map; import java.util.function.Predicate; -import ortus.boxlang.runtime.components.Component; import ortus.boxlang.runtime.scopes.IScope; import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.types.IStruct; @@ -142,10 +141,6 @@ public Object invokeFunction( Object function ) { return getParent().invokeFunction( function ); } - public Component.BodyResult invokeComponent( Key name, IStruct attributes, Component.ComponentBody componentBody ) { - return getParent().invokeComponent( name, attributes, componentBody ); - } - public IBoxContext pushComponent( IStruct executionState ) { return getParent().pushComponent( executionState ); } diff --git a/src/test/java/TestCases/components/BoxTemplateTest.java b/src/test/java/TestCases/components/BoxTemplateTest.java index 4a712fdab..29bffbb94 100644 --- a/src/test/java/TestCases/components/BoxTemplateTest.java +++ b/src/test/java/TestCases/components/BoxTemplateTest.java @@ -1312,4 +1312,18 @@ public void testLessThanBeforeTag() { assertThat( variables.getAsString( Key.output ).trim() ).isEqualTo( "<" ); } + @Test + public void testAccessBXCatchInOutput() { + instance.executeSource( + """ + + + + #bxcatch.message#sdf + + + """, + context, BoxSourceType.BOXTEMPLATE ); + } + } From 41e8c6fcdcec4bb3c2c91fab88dbcbffab7f2ff1 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Sat, 11 Jan 2025 14:09:53 -0600 Subject: [PATCH 110/161] BL-923 --- .../compiler/toolchain/BoxVisitor.java | 2 +- .../boxlang/compiler/toolchain/CFVisitor.java | 2 +- .../TestCases/phase2/UDFFunctionTest.java | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java index 93e817e87..ecf3522c0 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java @@ -1089,7 +1089,7 @@ private BoxFunctionDeclaration buildFunction( List annotations, List annToRemove ) { - annotations.stream().filter( pre -> pre.getKey().getValue().toLowerCase().startsWith( argDeclaration.getName().toLowerCase() ) ).forEach( pre -> { + annotations.stream().filter( pre -> pre.getKey().getValue().toLowerCase().startsWith( argDeclaration.getName().toLowerCase() + "." ) ).forEach( pre -> { String preName = pre.getKey().getValue(); BoxFQN key = new BoxFQN( preName.substring( pre.getKey().getValue().indexOf( "." ) + 1 ), pre.getPosition(), pre.getSourceText() ); argDeclaration.getAnnotations().add( new BoxAnnotation( key, pre.getValue(), pre.getPosition(), pre.getSourceText() ) ); diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/CFVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/CFVisitor.java index 8bc5f3ce6..7dc7604cc 100644 --- a/src/main/java/ortus/boxlang/compiler/toolchain/CFVisitor.java +++ b/src/main/java/ortus/boxlang/compiler/toolchain/CFVisitor.java @@ -1043,7 +1043,7 @@ private BoxFunctionDeclaration buildFunction( List postAn } private void buildAnnotations( BoxArgumentDeclaration argDeclaration, List annotations, List annToRemove ) { - annotations.stream().filter( pre -> pre.getKey().getValue().toLowerCase().startsWith( argDeclaration.getName().toLowerCase() ) ).forEach( pre -> { + annotations.stream().filter( pre -> pre.getKey().getValue().toLowerCase().startsWith( argDeclaration.getName().toLowerCase() + "." ) ).forEach( pre -> { String preName = pre.getKey().getValue(); BoxFQN key = new BoxFQN( preName.substring( pre.getKey().getValue().indexOf( "." ) + 1 ), pre.getPosition(), pre.getSourceText() ); argDeclaration.getAnnotations().add( new BoxAnnotation( key, pre.getValue(), pre.getPosition(), pre.getSourceText() ) ); diff --git a/src/test/java/TestCases/phase2/UDFFunctionTest.java b/src/test/java/TestCases/phase2/UDFFunctionTest.java index a35027a8c..1e14b8310 100644 --- a/src/test/java/TestCases/phase2/UDFFunctionTest.java +++ b/src/test/java/TestCases/phase2/UDFFunctionTest.java @@ -1029,4 +1029,33 @@ function foo() { assertThat( variables.getAsStruct( Key.of( "localRef" ) ).get( Key.of( "arguments" ) ) ).isEqualTo( "hello" ); } + @Test + public void testFunctionMetadata() { + instance.executeSource( + """ + /** + * @event.brad wood + * @eventgavin pickin + */ + @event.luis "majano" + @eventjon "clausen" + void function preEvent( event ) eventesme="acevado" {} + + result = getMetadata( preEvent ); + """, + context ); + + IStruct meta = variables.getAsStruct( result ); + assertThat( meta ).containsKey( Key.of( "documentation" ) ); + assertThat( meta ).containsKey( Key.of( "annotations" ) ); + assertThat( meta ).containsKey( Key.of( "parameters" ) ); + assertThat( meta.getAsStruct( Key.of( "documentation" ) ).get( Key.of( "eventgavin" ) ).toString() ).isEqualTo( "pickin" ); + assertThat( meta.getAsStruct( Key.of( "annotations" ) ).get( Key.of( "eventjon" ) ).toString() ).isEqualTo( "clausen" ); + assertThat( meta.getAsStruct( Key.of( "annotations" ) ).get( Key.of( "eventesme" ) ).toString() ).isEqualTo( "acevado" ); + assertThat( meta.getAsArray( Key.of( "parameters" ) ).size() ).isEqualTo( 1 ); + IStruct paramMeta = ( Struct ) meta.getAsArray( Key.of( "parameters" ) ).get( 0 ); + assertThat( paramMeta.getAsStruct( Key.of( "documentation" ) ).get( Key.of( "brad" ) ).toString() ).isEqualTo( "wood" ); + assertThat( paramMeta.getAsStruct( Key.of( "annotations" ) ).get( Key.of( "luis" ) ).toString() ).isEqualTo( "majano" ); + } + } From 0e252a4a239aa6e013bcaed81cdc550707afb9be Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Sat, 11 Jan 2025 15:33:01 -0500 Subject: [PATCH 111/161] BL-919 Resolve - Implement the Quick hash algorithm for fast 64 bit hashes --- .../runtime/bifs/global/encryption/Hash.java | 19 ++++--- .../boxlang/runtime/util/EncryptionUtil.java | 52 +++++++++++++++++++ .../bifs/global/encryption/HashTest.java | 15 ++++++ 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/encryption/Hash.java b/src/main/java/ortus/boxlang/runtime/bifs/global/encryption/Hash.java index 1dadfba46..05c9763dd 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/encryption/Hash.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/encryption/Hash.java @@ -34,6 +34,7 @@ import ortus.boxlang.runtime.types.exceptions.BoxIOException; import ortus.boxlang.runtime.types.util.JSONUtil; import ortus.boxlang.runtime.util.EncryptionUtil; +import ortus.boxlang.runtime.dynamic.casters.StringCaster; @BoxBIF @BoxBIF( alias = "Hash40" ) @@ -45,6 +46,7 @@ public class Hash extends BIF { private static final String DEFAULT_ALGORITHM = "MD5"; + private static final String QUICK_ALGORITHM = "quick"; private static final String DEFAULT_ENCODING = "utf-8"; private static final Integer DEFAULT_ITERATIONS = 1; @@ -75,7 +77,7 @@ public Hash() { * * @argument.input The item to be hashed * - * @argument.algorithm The supported {@link java.security.MessageDigest } algorithm (case-insensitive) + * @argument.algorithm The supported {@link java.security.MessageDigest } algorithm (case-insensitive) or "quick" for an insecure 64-bit hash * * @argument.encoding Applicable to strings ( default "utf-8" ) * @@ -83,12 +85,17 @@ public Hash() { */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { hashItem = arguments.get( Key.input ); - byte[] hashBytes = null; - Integer iterations = arguments.getAsInteger( Key.numIterations ); - String algorithm = arguments.getAsString( Key.algorithm ); - String charset = arguments.getAsString( Key.encoding ); + byte[] hashBytes = null; + Integer iterations = arguments.getAsInteger( Key.numIterations ); + String algorithm = arguments.getAsString( Key.algorithm ); + String charset = arguments.getAsString( Key.encoding ); + boolean isQuickAlgorithm = algorithm.trim().toLowerCase() == QUICK_ALGORITHM; + + if ( isQuickAlgorithm ) { + return EncryptionUtil.generate64BitHash( StringCaster.cast( hashItem ), 16 ); + } - Key bifMethodKey = arguments.getAsKey( BIF.__functionName ); + Key bifMethodKey = arguments.getAsKey( BIF.__functionName ); if ( bifMethodKey.equals( Key.hash40 ) ) { algorithm = "SHA1"; } diff --git a/src/main/java/ortus/boxlang/runtime/util/EncryptionUtil.java b/src/main/java/ortus/boxlang/runtime/util/EncryptionUtil.java index f2a7cb757..74ffd75e5 100644 --- a/src/main/java/ortus/boxlang/runtime/util/EncryptionUtil.java +++ b/src/main/java/ortus/boxlang/runtime/util/EncryptionUtil.java @@ -108,6 +108,13 @@ public final class EncryptionUtil { */ private static HashMap randomStore = new HashMap(); + /** + * Quick 64 bit hash properties + */ + private static final long[] byteTable = generateHashLookupTable(); + private static final long HSTART = 0xBB40E64DA205B064L; + private static final long HMULT = 7664345821815920749L; + /** * Supported key algorithms * key factory algorithms @@ -810,4 +817,49 @@ private static boolean isECBMode( String algorithm ) { return algorithmParts.length > 1 && algorithmParts[ 1 ].equals( "ECB" ); } + /** + * Creates an insecure but very fast 64 bit hash of a string + * + * @param hashItem the string to hash + */ + public static String generate64BitHash( CharSequence hashItem ) { + return generate64BitHash( hashItem, Character.MAX_RADIX ); + } + + /** + * Creates an insecure but very fast 64 bit hash of a string + * + * @param hashItem the string to hash + * @param size the radix to use for the final length of the hash + */ + public static String generate64BitHash( CharSequence hashItem, int size ) { + long h = HSTART; + final long hmult = HMULT; + final long[] ht = byteTable; + final int len = hashItem.length(); + for ( int i = 0; i < len; i++ ) { + char ch = hashItem.charAt( i ); + h = ( h * hmult ) ^ ht[ ch & 0xff ]; + h = ( h * hmult ) ^ ht[ ( ch >>> 8 ) & 0xff ]; + } + return Long.toString( h < 0 ? 0 - h : h, size ); + } + + /** + * Creates a lookup table for 64 bit hashes + */ + private static final long[] generateHashLookupTable() { + long[] _byteTable = new long[ 256 ]; + long h = 0x544B2FBACAAF1684L; + for ( int i = 0; i < 256; i++ ) { + for ( int j = 0; j < 31; j++ ) { + h = ( h >>> 7 ) ^ h; + h = ( h << 11 ) ^ h; + h = ( h >>> 10 ) ^ h; + } + _byteTable[ i ] = h; + } + return _byteTable; + } + } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/encryption/HashTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/encryption/HashTest.java index adb5b934b..e9419a085 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/encryption/HashTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/encryption/HashTest.java @@ -35,6 +35,9 @@ import ortus.boxlang.runtime.scopes.Key; import ortus.boxlang.runtime.scopes.VariablesScope; import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException; +import ortus.boxlang.runtime.dynamic.casters.IntegerCaster; + +import java.util.Objects; public class HashTest { @@ -177,6 +180,18 @@ public void testMemberString() { assertThat( variables.getAsString( Key.of( "result" ) ).length() ).isEqualTo( 32 ); } + @DisplayName( "Tests quick hash algorithm" ) + @Test + public void testQuickHash() { + instance.executeSource( + """ + result = hash( "foo", "quick" ); + """, + context ); + assertThat( variables.getAsString( Key.of( "result" ) ).length() ).isEqualTo( 16 ); + assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "4d780c14822d4653" ); + } + @DisplayName( "It will provide a consistent output and handle iterations correctly" ) @Test public void testIterations() { From 2e9306f2df645b5b425c0616c407f9c553c5d05e Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Thu, 9 Jan 2025 09:32:01 +0000 Subject: [PATCH 112/161] fix docs --- .../boxlang/runtime/bifs/global/system/GetClassMetadata.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetClassMetadata.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetClassMetadata.java index c0fa0e509..af3c68d80 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetClassMetadata.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/GetClassMetadata.java @@ -48,6 +48,7 @@ public GetClassMetadata() { * @param context The context in which the BIF is being invoked. * @param arguments Argument scope for the BIF. * + * @argument.path The path to the class or interface or an instance of the object to get the metadata for. */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { Object path = arguments.get( Key.path ); From 7cee4add94e8d60dc1d4f9482178755bab44ae1e Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Thu, 9 Jan 2025 09:40:23 +0000 Subject: [PATCH 113/161] BL-621 #resolve Allow a productivity hack to add ability to pass queries to some struct functions by taking the first row and converting the query to a struct. --- .../runtime/dynamic/casters/StructCaster.java | 7 ++++++ .../global/struct/StructKeyExistsTest.java | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java index 3ddde6e60..44bab681e 100644 --- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java +++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java @@ -22,6 +22,8 @@ import ortus.boxlang.runtime.interop.DynamicObject; import ortus.boxlang.runtime.scopes.ArgumentsScope; import ortus.boxlang.runtime.types.IStruct; +import ortus.boxlang.runtime.types.Query; +import ortus.boxlang.runtime.types.Struct; import ortus.boxlang.runtime.types.StructMapWrapper; import ortus.boxlang.runtime.types.exceptions.BoxCastException; import ortus.boxlang.runtime.types.exceptions.ExceptionUtil; @@ -90,6 +92,11 @@ public static IStruct cast( Object object, Boolean fail ) { return StructMapWrapper.of( ( Map ) map ); } + // Special Productivity Hack: If it's a Query object, take the first row and return it as a struct + if ( object instanceof Query query ) { + return query.isEmpty() ? new Struct() : query.getRowAsStruct( 0 ); + } + if ( fail ) { throw new BoxCastException( String.format( "Can't cast [%s] to a Struct.", object.getClass().getName() ) diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructKeyExistsTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructKeyExistsTest.java index 356b2d82c..dc381f21f 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructKeyExistsTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructKeyExistsTest.java @@ -19,6 +19,7 @@ package ortus.boxlang.runtime.bifs.global.struct; +import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -108,4 +109,28 @@ public void testNull() { assertTrue( variables.getAsBoolean( result ) ); } + @DisplayName( "It can work on a query row object" ) + @Test + public void testQueryRow() { + instance.executeSource( + """ + q = queryNew("col1,col2","string, integer", [ "foo", 42 ]); + result = structKeyExists( q, "col1" ); + """, + context ); + assertThat( variables.getAsBoolean( result ) ).isTrue(); + } + + @DisplayName( "It can work on a query row object that's empty and return an empty struct" ) + @Test + public void testEmptyQueryRow() { + instance.executeSource( + """ + q = queryNew("col1,col2","string, integer"); + result = structKeyExists( q, "col1" ); + """, + context ); + assertThat( variables.getAsBoolean( result ) ).isFalse(); + } + } From 9de4292ff4fd3b61a7aadfa2a72b6eea34bb92f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:16:43 +0000 Subject: [PATCH 114/161] Bump com.diffplug.spotless from 6.25.0 to 7.0.1 Bumps com.diffplug.spotless from 6.25.0 to 7.0.1. --- updated-dependencies: - dependency-name: com.diffplug.spotless dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ebc888ba2..7c0cd79b2 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ plugins { id "application" id 'antlr' // For source code formatting - id "com.diffplug.spotless" version "6.25.0" + id "com.diffplug.spotless" version "7.0.1" // For building shadow jars with jdk 17 ONLY //id 'com.github.johnrengelman.shadow' version '8.1.1' // For building shadow jars using JDK 21 +, they had to fork From 0914143f2ea0ee52772bee898f7e2a34315328fb Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Mon, 13 Jan 2025 10:32:17 -0500 Subject: [PATCH 115/161] BL-928 Resolve - Change invoke first arg to object instead of instance --- .../java/ortus/boxlang/runtime/bifs/global/system/Invoke.java | 4 ++-- .../ortus/boxlang/runtime/bifs/global/system/InvokeTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java index 6f790b317..99ca2cd5a 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java @@ -43,7 +43,7 @@ public class Invoke extends BIF { public Invoke() { super(); declaredArguments = new Argument[] { - new Argument( true, "any", Key.instance ), + new Argument( true, "any", Key.object ), new Argument( true, "string", Key.methodname ), new Argument( false, "any", Key.arguments ) }; @@ -63,7 +63,7 @@ public Invoke() { * @argument.arguments An array of positional arguments or a struct of named arguments to pass into the method. */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { - Object instance = arguments.get( Key.instance ); + Object instance = arguments.get( Key.object ); Key methodname = Key.of( arguments.getAsString( Key.methodname ) ); Object args = arguments.get( Key.arguments ); IStruct argCollection = Struct.of(); diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java index dc6d620db..ea3d2a1e4 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java @@ -106,7 +106,7 @@ function meh( x=3 ) { variables.result = arguments; } createArgs() - invoke( instance="", methodName="meh", arguments=args ); + invoke( object="", methodName="meh", arguments=args ); """, context ); @@ -129,7 +129,7 @@ function meh( a ) { variables.result = arguments; } createArgs('hello world') - invoke( instance="", methodName="meh", arguments=args ); + invoke( object="", methodName="meh", arguments=args ); """, context ); From d4fcfc514386132a4630a703ae4ca017999bdef2 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 13 Jan 2025 16:42:54 +0100 Subject: [PATCH 116/161] BL-621 test updates --- .../bifs/global/conversion/ToUnmodifiable.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToUnmodifiable.java b/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToUnmodifiable.java index 379ddba12..5d398fbe7 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToUnmodifiable.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToUnmodifiable.java @@ -64,17 +64,20 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { if ( castedArray.wasSuccessful() ) { return castedArray.get().toUnmodifiable(); } + + // Queries + var castedQuery = QueryCaster.attempt( inputValue ); + if ( castedQuery.wasSuccessful() ) { + return castedQuery.get().toUnmodifiable(); + } + // Structs var castedStruct = StructCaster.attempt( inputValue ); if ( castedStruct.wasSuccessful() ) { // This cast is not safe. Need to add .toUnmodifiable() to the IStruct interface return ( ( Struct ) castedStruct.get() ).toUnmodifiable(); } - // Queries - var castedQuery = QueryCaster.attempt( inputValue ); - if ( castedQuery.wasSuccessful() ) { - return castedQuery.get().toUnmodifiable(); - } + // Exceptions throw new BoxRuntimeException( "Cannot convert value to Unmodifiable type as it is not a struct, array or query" ); From 2049bf087a9a333865a78031a736f5f26981d9bc Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Mon, 13 Jan 2025 10:53:54 -0500 Subject: [PATCH 117/161] BL-928 - Change methodname to method --- .../boxlang/runtime/bifs/global/system/Invoke.java | 12 ++++++------ .../runtime/bifs/global/system/InvokeTest.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java index 99ca2cd5a..d69ac6df1 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java @@ -44,7 +44,7 @@ public Invoke() { super(); declaredArguments = new Argument[] { new Argument( true, "any", Key.object ), - new Argument( true, "string", Key.methodname ), + new Argument( true, "string", Key.method ), new Argument( false, "any", Key.arguments ) }; } @@ -58,13 +58,13 @@ public Invoke() { * @argument.instance Instance of a Box Class or name of one to instantiate. If an empty string is provided, the method will be invoked within the * same template or Box Class. * - * @argument.methodname The name of the method to invoke + * @argument.method The name of the method to invoke * * @argument.arguments An array of positional arguments or a struct of named arguments to pass into the method. */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { Object instance = arguments.get( Key.object ); - Key methodname = Key.of( arguments.getAsString( Key.methodname ) ); + Key method = Key.of( arguments.getAsString( Key.method ) ); Object args = arguments.get( Key.arguments ); IStruct argCollection = Struct.of(); @@ -76,7 +76,7 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { CastAttempt stringCasterAttempt = StringCaster.attempt( instance ); // Empty string just calls local function in the existing context (box class or template) if ( stringCasterAttempt.wasSuccessful() && stringCasterAttempt.get().isEmpty() ) { - return context.invokeFunction( methodname, argCollection ); + return context.invokeFunction( method, argCollection ); } // If we had a non-empty string, create the Box Class instance @@ -93,7 +93,7 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { // ALERT! // Special Case: If the instance is a DynamicObject and the method is "init", we need to call the constructor - if ( actualInstance instanceof DynamicObject castedDo && methodname.equals( Key.init ) ) { + if ( actualInstance instanceof DynamicObject castedDo && method.equals( Key.init ) ) { // The incoming args must be an array or throw an exception if ( ! ( args instanceof Array castedArray ) ) { throw new BoxValidationException( "The arguments must be an array in order to execute the Java constructor." ); @@ -101,7 +101,7 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { return castedDo.invokeConstructor( context, castedArray.toArray() ); } - return actualInstance.dereferenceAndInvoke( context, methodname, argCollection, false ); + return actualInstance.dereferenceAndInvoke( context, method, argCollection, false ); } } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java index ea3d2a1e4..51d8bc277 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/InvokeTest.java @@ -106,7 +106,7 @@ function meh( x=3 ) { variables.result = arguments; } createArgs() - invoke( object="", methodName="meh", arguments=args ); + invoke( object="", method="meh", arguments=args ); """, context ); @@ -129,7 +129,7 @@ function meh( a ) { variables.result = arguments; } createArgs('hello world') - invoke( object="", methodName="meh", arguments=args ); + invoke( object="", method="meh", arguments=args ); """, context ); From 234a02418096ae4cd15f68dbbbd5b5c03ed50555 Mon Sep 17 00:00:00 2001 From: jclausen <5255645+jclausen@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:55:26 +0000 Subject: [PATCH 118/161] Apply cfformat changes --- .../java/ortus/boxlang/runtime/bifs/global/system/Invoke.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java index d69ac6df1..03a53e514 100644 --- a/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java +++ b/src/main/java/ortus/boxlang/runtime/bifs/global/system/Invoke.java @@ -64,7 +64,7 @@ public Invoke() { */ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) { Object instance = arguments.get( Key.object ); - Key method = Key.of( arguments.getAsString( Key.method ) ); + Key method = Key.of( arguments.getAsString( Key.method ) ); Object args = arguments.get( Key.arguments ); IStruct argCollection = Struct.of(); From 516ed4dc716f10d08ffcf0f634379c1ae6a8c35c Mon Sep 17 00:00:00 2001 From: Jon Clausen Date: Mon, 13 Jan 2025 11:32:50 -0500 Subject: [PATCH 119/161] BL-930 Resolve - fixes for top level keys and null values --- .../runtime/types/util/StructUtil.java | 15 +++------ .../bifs/global/struct/StructFindKeyTest.java | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java b/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java index aebd6a513..39b25d3fc 100644 --- a/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java @@ -448,7 +448,8 @@ public static Stream findKey( IStruct struct, String key ) { .stream() .filter( entry -> { String stringKey = entry.getKey().getName().toLowerCase(); - return StringUtils.right( stringKey, keyLength ).equals( key.toLowerCase() ); + // We look for the key at the end of the string ( nested ) or at the beginning of the string for top-level keys + return StringUtils.right( stringKey, keyLength ).equals( key.toLowerCase() ) || StringUtils.left( stringKey, keyLength ).equals( key.toLowerCase() ); } ) .map( entry -> { Struct returnStruct = new Struct( Struct.TYPES.LINKED ); @@ -573,17 +574,9 @@ public static IStruct toFlatMap( IStruct struct ) { return new Struct( struct.getType(), struct.entrySet().stream() + .filter( entry -> entry.getValue() != null ) .flatMap( StructUtil::flattenEntry ) - .collect( - Collectors.toMap( - entry -> entry.getKey(), - entry -> entry.getValue(), - ( v1, v2 ) -> { - throw new BoxRuntimeException( "An exception occurred while flattening the struct" ); - }, - LinkedHashMap::new - ) - ) + .collect( LinkedHashMap::new, ( m, entry ) -> m.put( entry.getKey(), entry.getValue() ), LinkedHashMap::putAll ) ); } diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java index a54202548..87e0cbaa4 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java @@ -195,4 +195,35 @@ public void testMemberFunction() { assertEquals( StructCaster.cast( variables.getAsArray( result ).get( 0 ) ).get( Key.value ), 5 ); } + @DisplayName( "It tests the BIF StructFindKey with a null values in the struct" ) + @Test + public void testsFindKeyWithNulls() { + //@formatter:off + instance.executeSource( + """ + myStruct = { + horse: nullValue(), + bird: { + total: nullValue() + }, + cow: { + total: 12 + }, + pig: { + total: 5 + }, + cat: { + total: 3 + } + }; + result = StructFindKey( myStruct, "pig.total" ); + resultTop = StructFindKey( myStruct, "cat" ); + """, + context ); + //@formatter:on + assertTrue( variables.get( result ) instanceof Array ); + assertEquals( 1, variables.getAsArray( result ).size() ); + assertEquals( 1, variables.getAsArray( Key.of( "resultTop") ).size() ); + } + } From 9646f0fae423d5489e858e366f63e32b4066c93a Mon Sep 17 00:00:00 2001 From: jclausen <5255645+jclausen@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:42:46 +0000 Subject: [PATCH 120/161] Apply cfformat changes --- .../java/ortus/boxlang/runtime/types/util/StructUtil.java | 5 +++-- .../runtime/bifs/global/struct/StructFindKeyTest.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java b/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java index 39b25d3fc..bfe6c2c47 100644 --- a/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java +++ b/src/main/java/ortus/boxlang/runtime/types/util/StructUtil.java @@ -448,8 +448,9 @@ public static Stream findKey( IStruct struct, String key ) { .stream() .filter( entry -> { String stringKey = entry.getKey().getName().toLowerCase(); - // We look for the key at the end of the string ( nested ) or at the beginning of the string for top-level keys - return StringUtils.right( stringKey, keyLength ).equals( key.toLowerCase() ) || StringUtils.left( stringKey, keyLength ).equals( key.toLowerCase() ); + // We look for the key at the end of the string ( nested ) or at the beginning of the string for top-level keys + return StringUtils.right( stringKey, keyLength ).equals( key.toLowerCase() ) + || StringUtils.left( stringKey, keyLength ).equals( key.toLowerCase() ); } ) .map( entry -> { Struct returnStruct = new Struct( Struct.TYPES.LINKED ); diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java index 87e0cbaa4..be95885ca 100644 --- a/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java +++ b/src/test/java/ortus/boxlang/runtime/bifs/global/struct/StructFindKeyTest.java @@ -223,7 +223,7 @@ public void testsFindKeyWithNulls() { //@formatter:on assertTrue( variables.get( result ) instanceof Array ); assertEquals( 1, variables.getAsArray( result ).size() ); - assertEquals( 1, variables.getAsArray( Key.of( "resultTop") ).size() ); + assertEquals( 1, variables.getAsArray( Key.of( "resultTop" ) ).size() ); } } From 7a75291e9e9f46d7b440ff6ee011ad90baad811b Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 13 Jan 2025 12:28:54 -0600 Subject: [PATCH 121/161] BL-918 --- .../compiler/parser/BoxTemplateParser.java | 6 +++- .../boxlang/compiler/parser/CFParser.java | 6 +++- .../runtime/components/system/LoopTest.java | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/java/ortus/boxlang/compiler/parser/BoxTemplateParser.java b/src/main/java/ortus/boxlang/compiler/parser/BoxTemplateParser.java index 47f38cf61..0930067e2 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/BoxTemplateParser.java +++ b/src/main/java/ortus/boxlang/compiler/parser/BoxTemplateParser.java @@ -490,9 +490,13 @@ private BoxStatement toAst( File file, GenericOpenComponentContext node ) { for ( var attr : attributes ) { if ( attr.getKey().getValue().equalsIgnoreCase( "condition" ) ) { BoxExpression condition = attr.getValue(); + // parse as CF script expression and update value + // In reality, we could just re-parse the source text for all expression types, but there's really no need unless it was a string or interpolated string. if ( condition instanceof BoxStringLiteral str ) { - // parse as Box script expression and update value condition = parseBoxExpression( str.getValue(), condition.getPosition() ); + } else if ( condition instanceof BoxStringInterpolation stri ) { + // Strip "" from around the string + condition = parseBoxExpression( stri.getSourceText().substring( 1, stri.getSourceText().length() - 1 ), condition.getPosition() ); } BoxExpression newCondition = new BoxClosure( List.of(), diff --git a/src/main/java/ortus/boxlang/compiler/parser/CFParser.java b/src/main/java/ortus/boxlang/compiler/parser/CFParser.java index 122dbf50f..0a3c58a4d 100644 --- a/src/main/java/ortus/boxlang/compiler/parser/CFParser.java +++ b/src/main/java/ortus/boxlang/compiler/parser/CFParser.java @@ -896,9 +896,13 @@ private BoxStatement toAst( File file, Template_genericOpenComponentContext node for ( var attr : attributes ) { if ( attr.getKey().getValue().equalsIgnoreCase( "condition" ) ) { BoxExpression condition = attr.getValue(); + // parse as CF script expression and update value + // In reality, we could just re-parse the source text for all expression types, but there's really no need unless it was a string or interpolated string. if ( condition instanceof BoxStringLiteral str ) { - // parse as CF script expression and update value condition = parseCFExpression( str.getValue(), condition.getPosition() ); + } else if ( condition instanceof BoxStringInterpolation stri ) { + // Strip "" from around the string + condition = parseCFExpression( stri.getSourceText().substring( 1, stri.getSourceText().length() - 1 ), condition.getPosition() ); } BoxExpression newCondition = new BoxClosure( List.of(), diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java index ab3c2ef82..01e6910ca 100644 --- a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java +++ b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java @@ -456,4 +456,32 @@ public void testLoopToFromDecimalStep() { assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "1,2.5,4,5.5,7,8.5,10" ); } + @Test + public void testLoopConditionMixCF() { + instance.executeSource( + """ + + + + #i# + + + """, + context, BoxSourceType.CFTEMPLATE ); + } + + @Test + public void testLoopConditionMix() { + instance.executeSource( + """ + + + + #i# + + + """, + context, BoxSourceType.BOXTEMPLATE ); + } + } From 341a3951917c5045753da06121bb737392d88978 Mon Sep 17 00:00:00 2001 From: Brad Wood Date: Mon, 13 Jan 2025 14:39:43 -0600 Subject: [PATCH 122/161] BL-926 --- src/test/java/TestCases/phase3/ASMError.cfc | 7 +++++++ src/test/java/TestCases/phase3/ClassTest.java | 10 ++++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/test/java/TestCases/phase3/ASMError.cfc diff --git a/src/test/java/TestCases/phase3/ASMError.cfc b/src/test/java/TestCases/phase3/ASMError.cfc new file mode 100644 index 000000000..24cd42ddc --- /dev/null +++ b/src/test/java/TestCases/phase3/ASMError.cfc @@ -0,0 +1,7 @@ +component { + + do { + break; + } while ( false ); + +} \ No newline at end of file diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java index eb725a9ea..bb8a28522 100644 --- a/src/test/java/TestCases/phase3/ClassTest.java +++ b/src/test/java/TestCases/phase3/ClassTest.java @@ -1645,4 +1645,14 @@ public void testPropertiesNotInheritedInMetadata() { } + @Test + @Disabled( "BL-926" ) + public void testUserASMError() { + instance.executeSource( + """ + new src.test.java.TestCases.phase3.ASMError() + """, + context ); + } + } From ae6d9861661a736db120040012fe86615db53279 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 13 Jan 2025 22:13:23 +0100 Subject: [PATCH 123/161] make sure we reuse the listener and not create one --- .../java/ortus/boxlang/runtime/application/Application.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ortus/boxlang/runtime/application/Application.java b/src/main/java/ortus/boxlang/runtime/application/Application.java index 6af564374..9f6637f99 100644 --- a/src/main/java/ortus/boxlang/runtime/application/Application.java +++ b/src/main/java/ortus/boxlang/runtime/application/Application.java @@ -506,7 +506,7 @@ public synchronized void shutdown( boolean force ) { } // Announce it globally - RequestBoxContext requestContext = new ScriptingRequestBoxContext( BoxRuntime.getInstance().getRuntimeContext() ); + RequestBoxContext requestContext = this.getStartingListener().getRequestContext(); BoxRuntime.getInstance().getInterceptorService().announce( Key.onApplicationEnd, Struct.of( "application", this, "context", requestContext From 983654067c0d2d270276d57c81be96aa7d9e2610 Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 13 Jan 2025 22:13:37 +0100 Subject: [PATCH 124/161] add a shutdown to the interface --- .../java/ortus/boxlang/runtime/context/IBoxContext.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/ortus/boxlang/runtime/context/IBoxContext.java b/src/main/java/ortus/boxlang/runtime/context/IBoxContext.java index 1cb3d6b92..41b5716e3 100644 --- a/src/main/java/ortus/boxlang/runtime/context/IBoxContext.java +++ b/src/main/java/ortus/boxlang/runtime/context/IBoxContext.java @@ -687,6 +687,13 @@ public Key[] getAssignmentKeys( Key... keys ) { */ public ModuleRecord getModuleRecord( Key name ); + /** + * This is an optional method on contexts which require shutdown outside of its constructor + */ + public default void shutdown() { + // Default is nothing + } + /** * This is an optional method on contexts which require startup outside of its constructor */ From 2e255d943cebb023150c90d787f838cbdb745c4d Mon Sep 17 00:00:00 2001 From: Luis Majano Date: Mon, 13 Jan 2025 22:29:39 +0100 Subject: [PATCH 125/161] changing defaults for datasources according to best practices in the hikari forums and research --- coldbox.sh | 3 + .../config/segments/DatasourceConfig.java | 156 +++++++++++------- .../ortus/boxlang/runtime/scopes/Key.java | 1 + 3 files changed, 97 insertions(+), 63 deletions(-) diff --git a/coldbox.sh b/coldbox.sh index c7d34378f..e85559925 100755 --- a/coldbox.sh +++ b/coldbox.sh @@ -1,9 +1,12 @@ #!/bin/sh cd ~/Sites/projects/boxlang +git pull --rebase --autostash gradle build -x test -x javadoc cd ~/Sites/projects/boxlang-web-support +git pull --rebase --autostash gradle build -x test -x javadoc cd ~/Sites/projects/boxlang-servlet +git pull --rebase --autostash gradle buildRuntime diff --git a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java index 69c8d7de1..c81345d0c 100644 --- a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java +++ b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java @@ -42,7 +42,8 @@ * A BoxLang datasource configuration. * *

    - * Inside a boxlang.json configuration file, a single datasource configuration might look something like this: + * Inside a boxlang.json configuration file, a single datasource configuration + * might look something like this: * * 

      * "myMysql": {
@@ -102,13 +103,13 @@ public class DatasourceConfig implements Comparable, IConfigSe
 	    // Adobe CF notation (url)
 	    Key.URL,
 	    // HikariConfig notation
-	    Key.jdbcURL
-	);
+	    Key.jdbcURL );
 
 	/**
 	 * BoxLang Datasource Default configuration values
 	 * These are applied to ALL datasources
-	 * Please note that most of them rely on Hikari defaults but we use seconds in BoxLang to match CFConfig standards
+	 * Please note that most of them rely on Hikari defaults but we use seconds in
+	 * BoxLang to match CFConfig standards
 	 * 
    
 	 * References
 	 * https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration
@@ -116,41 +117,55 @@ public class DatasourceConfig implements Comparable, IConfigSe
 	private static final IStruct	DEFAULTS						= Struct.of(
 	    // The maximum number of connections.
 	    // Hikari: maximumPoolSize
-	    "maxConnections", 10,
+	    "maxConnections", 15,
 	    // The minimum number of connections
-	    // Hikari: minimumIdle
-	    "minConnections", 10,
-	    // Maximum time to wait for a successful connection, in seconds ( 1 Second )
-	    "connectionTimeout", 1,
-	    // The maximum number of idle time in seconds ( 10 Minutes = 600 )
-	    // • Refers to the maximum amount of time a connection can remain idle in the pool before it is eligible for eviction.
-	    // •If a connection is idle for longer than this time, it can be closed and removed from the pool, helping to free up resources.
-	    // • This setting only affects connections that are not in use and have exceeded the idle duration specified.
+	    // Hikari: minimumIdle, try to stick to 80% of maxConnections
+	    "minConnections", 12,
+	    // Maximum time to wait for a successful connection, in seconds ( 5 Seconds )
+	    "connectionTimeout", 5,
+	    // The maximum number of idle time in seconds ( 5 Minutes = 300 )
+	    // • Refers to the maximum amount of time a connection can remain idle in the
+	    // pool before it is eligible for eviction.
+	    // • If a connection is idle for longer than this time, it can be closed and
+	    // removed from the pool, helping to free up resources.
+	    // • This setting only affects connections that are not in use and have exceeded
+	    // the idle duration specified.
 	    // In Seconds
-	    "idleTimeout", 600,
+	    "idleTimeout", 300,
 	    // This property controls the maximum lifetime of a connection in the pool.
-	    // An in-use connection will never be retired, only when it is closed will it then be removed
-	    // We strongly recommend setting this value, and it should be several seconds shorter than any database
+	    // An in-use connection will never be retired, only when it is closed will it
+	    // then be removed
+	    // We strongly recommend setting this value, and it should be several seconds
+	    // shorter than any database
 	    // or infrastructure imposed connection time limit
 	    // 30 minutes by default = 1800000ms = 1800 seconds
+	    // Match this with your database server’s timeout for connections (e.g., MySQL’s
+	    // wait_timeout)
 	    // In Seconds
 	    "maxLifetime", 1800,
-	    // This property controls how frequently HikariCP will attempt to keep a connection alive, in order to prevent it from being timed out by the database
+	    // This property controls how frequently HikariCP will attempt to keep a
+	    // connection alive, in order to prevent it from being timed out by the database
 	    // or network infrastructure
-	    // This value must be less than the maxLifetime value. A "keepalive" will only occur on an idle connectionThis value must be less than the maxLifetime
-	    // value. A "keepalive" will only occur on an idle connection ( 10 Minutes = 600 seconds = 600,000 ms )
+	    // This value must be less than the maxLifetime value. A "keepalive" will only
+	    // occur on an idle connectionThis value must be less than the maxLifetime
+	    // value. A "keepalive" will only occur on an idle connection ( 10 Minutes = 600
+	    // seconds = 600,000 ms )
 	    // In Seconds
 	    "keepaliveTime", 600,
 	    // The default auto-commit state of connections created by this pool
 	    "autoCommit", true,
 	    // Register mbeans or not. By default, this is true
-	    // However, if you are using JMX, you can set this to true to get some additional monitoring information
-	    "registerMbeans", true
-	);
-
-	// List of keys to NOT set dynamically. All keys not in this list will use `addDataSourceProperty` to set the property and pass it to the JDBC driver.
+	    // However, if you are using JMX, you can set this to true to get some
+	    // additional monitoring information
+	    "registerMbeans", true,
+	    // Leak detection threshold in seconds
+	    "leakDetectionThreshold", 3 );
+
+	// List of keys to NOT set dynamically. All keys not in this list will use
+	// `addDataSourceProperty` to set the property and pass it to the JDBC driver.
 	// Please use the hikariConfig setters for any hikari-specific properties.
 	private List				RESERVED_CONNECTION_PROPERTIES	= List.of(
+	    Key.leakDetectionThreshold,
 	    Key.autoCommit,
 	    Key.connectionString,
 	    Key.connectionTestQuery,
@@ -169,8 +184,7 @@ public class DatasourceConfig implements Comparable, IConfigSe
 	    Key.password,
 	    Key.poolName,
 	    Key.port,
-	    Key.username
-	);
+	    Key.username );
 
 	/**
 	 * Logger
@@ -253,7 +267,8 @@ public DatasourceConfig( String name ) {
 	public static String discoverDriverFromJdbcUrl( String jdbcURL ) {
 		logger.debug( "Attempting to determine driver from JDBC URL: {}", jdbcURL );
 
-		// check that the URL is not empty, that it has at least one : and that it starts with jdbc:
+		// check that the URL is not empty, that it has at least one : and that it
+		// starts with jdbc:
 		if ( jdbcURL == null || jdbcURL.isEmpty() || !jdbcURL.contains( ":" ) || !jdbcURL.startsWith( "jdbc:" ) ) {
 			return "";
 		}
@@ -324,7 +339,8 @@ public Key getUniqueName() {
 	}
 
 	/**
-	 * Get the original name of the datasource - this is NOT unique and should not be used for identification.
+	 * Get the original name of the datasource - this is NOT unique and should not
+	 * be used for identification.
 	 *
 	 * @return The original name of the datasource.
 	 */
@@ -333,7 +349,8 @@ public String getOriginalName() {
 	}
 
 	/**
-	 * Processes the state of the configuration segment from the configuration struct.
+	 * Processes the state of the configuration segment from the configuration
+	 * struct.
 	 * 
    
 	 * Each segment is processed individually from the initial configuration struct.
 	 * This is so we can handle cascading overrides from configuration loading.
@@ -440,8 +457,7 @@ public IStruct asStruct() {
 		    "name", this.name.getName(),
 		    "onTheFly", this.onTheFly,
 		    "uniqueName", this.getUniqueName().getName(),
-		    "properties", new Struct( this.properties )
-		);
+		    "properties", new Struct( this.properties ) );
 	}
 
 	/**
@@ -459,7 +475,8 @@ public int hashCode() {
 	 *
 	 * @param otherConfig The other DatasourceConfig object to compare
 	 *
-	 * @return A negative integer, zero, or a positive integer as this object is less than, equal to, or greater than the specified object.
+	 * @return A negative integer, zero, or a positive integer as this object is
+	 *         less than, equal to, or greater than the specified object.
 	 */
 	@Override
 	public int compareTo( DatasourceConfig otherConfig ) {
@@ -488,29 +505,35 @@ public boolean equals( Object obj ) {
 	 * Build a HikariConfig object from the datasource properties configuration.
 	 *
 	 * 
      
-	 * 
    1. Configure HikariCP-specific properties, i.e. jdbcUrl, username, password, etc, using the appropriate
+	 * 
    2. Configure HikariCP-specific properties, i.e. jdbcUrl,
+	 * username, password, etc, using the appropriate
 	 * setter methods on the HikariConfig object.
      3. 
-	 * 
    3. Import all other properties as generic DataSource properties. Vendor-specific properties, i.e. for Derby, Oracle, etc, such as
+	 * 
    4. Import all other properties as generic DataSource properties.
+	 * Vendor-specific properties, i.e. for Derby, Oracle, etc, such as
 	 * "derby.locks.deadlockTimeout".
      5. 
 	 * 
 	 *
 	 */
 	public HikariConfig toHikariConfig() {
 
-		// At this point, if no driver can be determined from the 'driver' or 'type' keys or JDBC url key(s),
+		// At this point, if no driver can be determined from the 'driver' or 'type'
+		// keys or JDBC url key(s),
 		// we need to throw an exception because we can't proceed.
 		if ( this.properties.getOrDefault( Key.driver, "" ).toString().isBlank() ) {
-			throw new IllegalArgumentException( "Datasource configuration must contain a 'driver', or a valid JDBC connection string in 'url'." );
+			throw new IllegalArgumentException(
+			    "Datasource configuration must contain a 'driver', or a valid JDBC connection string in 'url'." );
 		}
 
 		DatasourceService	datasourceService	= BoxRuntime.getInstance().getDataSourceService();
 		HikariConfig		result				= new HikariConfig();
 		// If we can't find the driver, we default to the generic driver
-		IJDBCDriver			driverOrDefault		= datasourceService.hasDriver( getDriver() ) ? datasourceService.getDriver( getDriver() )
+		IJDBCDriver			driverOrDefault		= datasourceService.hasDriver( getDriver() )
+		    ? datasourceService.getDriver( getDriver() )
 		    : datasourceService.getGenericDriver();
 
 		// Incorporate the driver's default properties
-		driverOrDefault.getDefaultProperties().entrySet().stream().forEach( entry -> this.properties.putIfAbsent( entry.getKey(), entry.getValue() ) );
+		driverOrDefault.getDefaultProperties().entrySet().stream()
+		    .forEach( entry -> this.properties.putIfAbsent( entry.getKey(), entry.getValue() ) );
 
 		// Make sure the `custom` property is a struct: Normalize it
 		if ( this.properties.get( Key.custom ) instanceof String castedCustomParams ) {
@@ -518,7 +541,8 @@ public HikariConfig toHikariConfig() {
 		}
 		// Incorporate the driver's default 'custom' properties
 		IStruct customParams = this.properties.getAsStruct( Key.custom );
-		driverOrDefault.getDefaultCustomParams().entrySet().stream().forEach( entry -> customParams.putIfAbsent( entry.getKey(), entry.getValue() ) );
+		driverOrDefault.getDefaultCustomParams().entrySet().stream()
+		    .forEach( entry -> customParams.putIfAbsent( entry.getKey(), entry.getValue() ) );
 		this.properties.put( Key.custom, customParams );
 
 		// Build out the JDBC URL according to the driver chosen or url chosen
@@ -534,8 +558,7 @@ public HikariConfig toHikariConfig() {
 		// Connection timeouts in seconds, but Hikari uses milliseconds
 		if ( properties.containsKey( Key.connectionTimeout ) ) {
 			result.setConnectionTimeout(
-			    LongCaster.cast( properties.get( Key.connectionTimeout ), false ) * 1000
-			);
+			    LongCaster.cast( properties.get( Key.connectionTimeout ), false ) * 1000 );
 		}
 		if ( properties.containsKey( Key.minConnections ) ) {
 			result.setMinimumIdle( IntegerCaster.cast( properties.get( Key.minConnections ), false ) );
@@ -550,18 +573,15 @@ public HikariConfig toHikariConfig() {
 		}
 		if ( properties.containsKey( Key.idleTimeout ) ) {
 			result.setIdleTimeout(
-			    LongCaster.cast( properties.get( Key.idleTimeout ), false ) * 1000
-			);
+			    LongCaster.cast( properties.get( Key.idleTimeout ), false ) * 1000 );
 		}
 		if ( properties.containsKey( Key.keepaliveTime ) ) {
 			result.setKeepaliveTime(
-			    LongCaster.cast( properties.get( Key.keepaliveTime ), false ) * 1000
-			);
+			    LongCaster.cast( properties.get( Key.keepaliveTime ), false ) * 1000 );
 		}
 		if ( properties.containsKey( Key.maxLifetime ) ) {
 			result.setMaxLifetime(
-			    LongCaster.cast( properties.get( Key.maxLifetime ), false ) * 1000
-			);
+			    LongCaster.cast( properties.get( Key.maxLifetime ), false ) * 1000 );
 		}
 		if ( properties.containsKey( Key.connectionTestQuery ) ) {
 			result.setConnectionTestQuery( properties.getAsString( Key.connectionTestQuery ) );
@@ -575,6 +595,9 @@ public HikariConfig toHikariConfig() {
 		if ( properties.containsKey( Key.poolName ) ) {
 			result.setPoolName( properties.getAsString( Key.poolName ) );
 		}
+		if ( properties.containsKey( Key.leakDetectionThreshold ) ) {
+			result.setLeakDetectionThreshold( LongCaster.cast( properties.get( Key.leakDetectionThreshold ), false ) * 1000 );
+		}
 
 		// ADD NON-RESERVED PROPERTIES
 		// as Hikari properties
@@ -586,9 +609,11 @@ public HikariConfig toHikariConfig() {
 	}
 
 	/**
-	 * Retrieve the connection string from the properties, or build it from the appropriate driver module.
+	 * Retrieve the connection string from the properties, or build it from the
+	 * appropriate driver module.
 	 *
-	 * If any of these properties are found, they will be returned as-is, in the following order:
+	 * If any of these properties are found, they will be returned as-is, in the
+	 * following order:
 	 * 
        
 	 * 
      • connectionString
        • 
 	 * 
      • URL
        • 
@@ -596,28 +621,33 @@ public HikariConfig toHikariConfig() {
 	 * 
      • dsn - Special case used on placeholder replacements
        • 
 	 * 
      
 	 *
-	 * If none of these properties are found then we delegate to a registered driver in the
+	 * If none of these properties are found then we delegate to a registered driver
+	 * in the
 	 * datasource service. If none, can be found, we use the generic JDBC Driver.
 	 *
 	 * @param driver The JDBC driver to use
 	 *
-	 * @return JDBC connection string, e.g. jdbc:mysql://localhost:3306/foo?useSSL=false
+	 * @return JDBC connection string, e.g.
+	 *         jdbc:mysql://localhost:3306/foo?useSSL=false
 	 */
 	private String getOrBuildConnectionString( IJDBCDriver driver ) {
 		// 1. Attempt to find the connection string from the properties first.
 		String connectionString = replaceConnectionPlaceholders( getConnectionString() );
 
-		// 2. If the attempt was empty, then try to find the connection string from the DSN cfconfig element
+		// 2. If the attempt was empty, then try to find the connection string from the
+		// DSN cfconfig element
 		if ( connectionString.isEmpty() && this.properties.containsKey( Key.dsn ) ) {
 			connectionString = replaceConnectionPlaceholders( this.properties.getAsString( Key.dsn ) );
 		}
 
-		// 3. If the attempt was empty, then try to build the connection string from the driver
+		// 3. If the attempt was empty, then try to build the connection string from the
+		// driver
 		// This adds all the placeholders and custom parameters via the driver
 		if ( connectionString.isEmpty() ) {
 			connectionString = replaceConnectionPlaceholders( driver.buildConnectionURL( this ) );
 		} else {
-			connectionString = addCustomParams( connectionString, driver.getDefaultURIDelimiter(), driver.getDefaultDelimiter() );
+			connectionString = addCustomParams( connectionString, driver.getDefaultURIDelimiter(),
+			    driver.getDefaultDelimiter() );
 		}
 
 		// Finalize with custom params
@@ -625,7 +655,8 @@ private String getOrBuildConnectionString( IJDBCDriver driver ) {
 	}
 
 	/**
-	 * This method is used to incorporate custom parameters into the target connection string.
+	 * This method is used to incorporate custom parameters into the target
+	 * connection string.
 	 *
 	 * @param target       The target connection string
 	 * @param URIDelimiter The URI delimiter to use
@@ -655,7 +686,8 @@ public String addCustomParams( String target, String URIDelimiter, String delimi
 	}
 
 	/**
-	 * This method is used to incorporate custom parameters into the target connection string
+	 * This method is used to incorporate custom parameters into the target
+	 * connection string
 	 * Using default delimiters of ? and &
 	 *
 	 * @param target The target connection string
@@ -667,7 +699,8 @@ public String addCustomParams( String target ) {
 	}
 
 	/**
-	 * This method is used to replace placeholders in the connection string with the appropriate values.
+	 * This method is used to replace placeholders in the connection string with the
+	 * appropriate values.
 	 * 
      
 	 * The placeholders are:
 	 * 
        
@@ -689,16 +722,13 @@ private String replaceConnectionPlaceholders( String target ) {
 		// Replace placeholders
 		target	= target.replace(
 		    "{host}",
-		    StringCaster.cast( this.properties.getOrDefault( Key.host, "NOT_FOUND" ), true )
-		);
+		    StringCaster.cast( this.properties.getOrDefault( Key.host, "NOT_FOUND" ), true ) );
 		target	= target.replace(
 		    "{port}",
-		    StringCaster.cast( this.properties.getOrDefault( Key.port, 0 ), true )
-		);
+		    StringCaster.cast( this.properties.getOrDefault( Key.port, 0 ), true ) );
 		target	= target.replace(
 		    "{database}",
-		    StringCaster.cast( this.properties.getOrDefault( Key.database, "NOT_FOUND" ), true )
-		);
+		    StringCaster.cast( this.properties.getOrDefault( Key.database, "NOT_FOUND" ), true ) );
 
 		return target;
 	}
diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
index d0490dabb..6247977ca 100644
--- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java
+++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
@@ -427,6 +427,7 @@ public class Key implements Comparable, Serializable {
 	public static final Key		language							= Key.of( "language" );
 	public static final Key		lastVisit							= Key.of( "lastVisit" );
 	public static final Key		leaveIndex							= Key.of( "leaveIndex" );
+	public static final Key		leakDetectionThreshold							= Key.of( "leakDetectionThreshold" );
 	public static final Key		len									= Key.of( "len" );
 	public static final Key		length								= Key.of( "length" );
 	public static final Key		level								= Key.of( "level" );

From be7414c5a75517e5ce7ca8d7d674f8dde335dcbd Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 01:47:26 +0100
Subject: [PATCH 126/161] missing leak detection key

---
 src/main/java/ortus/boxlang/runtime/scopes/Key.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
index 6247977ca..805fbb38a 100644
--- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java
+++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
@@ -427,7 +427,7 @@ public class Key implements Comparable, Serializable {
 	public static final Key		language							= Key.of( "language" );
 	public static final Key		lastVisit							= Key.of( "lastVisit" );
 	public static final Key		leaveIndex							= Key.of( "leaveIndex" );
-	public static final Key		leakDetectionThreshold							= Key.of( "leakDetectionThreshold" );
+	public static final Key		leakDetectionThreshold				= Key.of( "leakDetectionThreshold" );
 	public static final Key		len									= Key.of( "len" );
 	public static final Key		length								= Key.of( "length" );
 	public static final Key		level								= Key.of( "level" );

From a5510bc1a075b20c6e59ee87e0dfd782c462e069 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 01:59:07 +0100
Subject: [PATCH 127/161] BL-932 #resolve toBinary() not lenient enough for
 decoding with line paddings

---
 .../bifs/global/conversion/ToBinary.java       | 18 ++++++++++++++----
 .../bifs/global/conversion/ToBinaryTest.java   | 12 ++++++++++++
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinary.java b/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinary.java
index 59907a6f4..850996820 100644
--- a/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinary.java
+++ b/src/main/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinary.java
@@ -17,6 +17,8 @@
  */
 package ortus.boxlang.runtime.bifs.global.conversion;
 
+import com.fasterxml.jackson.core.Base64Variants;
+
 import ortus.boxlang.runtime.bifs.BIF;
 import ortus.boxlang.runtime.bifs.BoxBIF;
 import ortus.boxlang.runtime.bifs.BoxMember;
@@ -48,16 +50,24 @@ public ToBinary() {
 	 * @param arguments Argument scope for the BIF.
 	 *
 	 * @argument.base64_or_object A string containing base64-encoded data.
-	 * 
+	 *
 	 */
 	public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
 		Object base64_or_object = arguments.get( Key.base64_or_object );
 
 		if ( base64_or_object instanceof byte[] b ) {
-			return base64_or_object;
+			return b;
+		}
+
+		String	string	= StringCaster.cast( base64_or_object ).trim();
+
+		// Add padding if necessary
+		int		padding	= string.length() % 4;
+		if ( padding != 0 ) {
+			string = string + "=".repeat( 4 - padding ); // Add padding
 		}
 
-		String string = StringCaster.cast( base64_or_object );
-		return java.util.Base64.getDecoder().decode( string );
+		// return java.util.Base64.getDecoder().decode( string );
+		return Base64Variants.getDefaultVariant().decode( string );
 	}
 }
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinaryTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinaryTest.java
index 47c66224b..0f5eab554 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinaryTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/conversion/ToBinaryTest.java
@@ -75,4 +75,16 @@ public void testCanConvertString() {
 		assertThat( new String( ( byte[] ) variables.get( result ) ) ).isEqualTo( "Hello World" );
 	}
 
+	@Test
+	public void testIt() {
+		// @formatter:off
+		instance.executeSource(
+		    """
+				result = toBinary( "rO0ABXNyACJvcnR1cy5ib3hsYW5nLnJ1bnRpbWUudHlwZXMuU3RydWN0AAAAAAAAAAECAARMAAMkYnh0ACpMb3J0dXMvYm94bGFuZy9ydW50aW1lL3R5cGVzL21ldGEvQm94TWV0YTtMAAlsaXN0ZW5lcnN0AA9MamF2YS91dGlsL01hcDtMAAR0eXBldAArTG9ydHVzL2JveGxhbmcvcnVudGltZS90eXBlcy9JU3RydWN0JFRZUEVTO0wAB3dyYXBwZWRxAH4AAnhwcHB+cgApb3J0dXMuYm94bGFuZy5ydW50aW1lLnR5cGVzLklTdHJ1Y3QkVFlQRVMAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdERUZBVUxUc3IAJmphdmEudXRpbC5jb25jdXJyZW50LkNvbmN1cnJlbnRIYXNoTWFwZJneEp2HKT0DAANJAAtzZWdtZW50TWFza0kADHNlZ21lbnRTaGlmdFsACHNlZ21lbnRzdAAxW0xqYXZhL3V0aWwvY29uY3VycmVudC9Db25jdXJyZW50SGFzaE1hcCRTZWdtZW50O3hwAAAADwAAABx1cgAxW0xqYXZhLnV0aWwuY29uY3VycmVudC5Db25jdXJyZW50SGFzaE1hcCRTZWdtZW50O1J3P0Eymzl0AgAAeHAAAAAQc3IALmphdmEudXRpbC5jb25jdXJyZW50LkNvbmN1cnJlbnRIYXNoTWFwJFNlZ21lbnQfNkyQWJMpPQIAAUYACmxvYWRGYWN0b3J4cgAoamF2YS51dGlsLmNvbmN1cnJlbnQubG9ja3MuUmVlbnRyYW50TG9ja2ZVqCwsyGrrAgABTAAEc3luY3QAL0xqYXZhL3V0aWwvY29uY3VycmVudC9sb2Nrcy9SZWVudHJhbnRMb2NrJFN5bmM7eHBzcgA0amF2YS51dGlsLmNvbmN1cnJlbnQubG9ja3MuUmVlbnRyYW50TG9jayROb25mYWlyU3luY2WIMudTe78LAgAAeHIALWphdmEudXRpbC5jb25jdXJyZW50LmxvY2tzLlJlZW50cmFudExvY2skU3luY7geopSqRFp8AgAAeHIANWphdmEudXRpbC5jb25jdXJyZW50LmxvY2tzLkFic3RyYWN0UXVldWVkU3luY2hyb25pemVyZlWoQ3U/UuMCAAFJAAVzdGF0ZXhyADZqYXZhLnV0aWwuY29uY3VycmVudC5sb2Nrcy5BYnN0cmFjdE93bmFibGVTeW5jaHJvbml6ZXIz36+5rW1vqQIAAHhwAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcgAgb3J0dXMuYm94bGFuZy5ydW50aW1lLnNjb3Blcy5LZXkAAAAAAAAAAQIABEkACGhhc2hDb2RlTAAEbmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wACm5hbWVOb0Nhc2VxAH4ANkwADW9yaWdpbmFsVmFsdWV0ABJMamF2YS9sYW5nL09iamVjdDt4cJ/IfZl0AAxsYXN0QWNjZXNzZWR0AAxMQVNUQUNDRVNTRURxAH4AOXNyACRvcnR1cy5ib3hsYW5nLnJ1bnRpbWUudHlwZXMuRGF0ZVRpbWUAAAAAAAAAAQIAAUwAB3dyYXBwZWR0ABlMamF2YS90aW1lL1pvbmVkRGF0ZVRpbWU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3FQYAAAfpAQ4AIyI0ZwoAAAcAA1VUQ3hzcQB+ADVoGgrIdAAHY3JlYXRlZHQAB0NSRUFURURxAH4AQXNxAH4AO3NxAH4APncVBgAAB+kBDgAjIjRm9ngABwADVVRDeHNxAH4ANdJHZpt0AAlpc0V4cGlyZWR0AAlJU0VYUElSRURxAH4ARnNyABFqYXZhLmxhbmcuQm9vbGVhbs0gcoDVnPruAgABWgAFdmFsdWV4cABzcQB+ADUAIddAdAAEaGl0c3QABEhJVFNxAH4AS3NyABFqYXZhLmxhbmcuSW50ZWdlchLioKT3gYc4AgABSQAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAABc3EAfgA1ipOXP3QABm9iamVjdHQABk9CSkVDVHEAfgBRc3EAfgAAcHBxAH4AB3NxAH4ACQAAAA8AAAAcdXEAfgAMAAAAEHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgAOc3EAfgASAAAAAD9AAABzcQB+AA5zcQB+ABIAAAAAP0AAAHNxAH4ADnNxAH4AEgAAAAA/QAAAc3EAfgA1AAD833QAA0FHRXEAfgB3cQB+AHdzcQB+AE0AAAAgc3EAfgA1ACRyi3QABE5BTUVxAH4AenEAfgB6dAAEbHVpc3BweHNxAH4ANXZ2cId0ABFsYXN0QWNjZXNzVGltZW91dHQAEUxBU1RBQ0NFU1NUSU1FT1VUcQB+AH1zcQB+AE0AAAAUc3EAfgA13HrZQXQAB3RpbWVvdXR0AAdUSU1FT1VUcQB+AIFxAH4Af3BweA==cHB4" )
+		    	writedump( result )
+	    	""",
+	    context );
+		// @formatter:on
+	}
+
 }

From 9f9354a295d8cf98bbb69acd97e6f220d510776a Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Mon, 13 Jan 2025 19:40:24 -0600
Subject: [PATCH 128/161] BL-933

---
 .../global/query/QueryRegisterFunction.java   | 82 +++++++++++++++++++
 .../runtime/jdbc/qoq/QoQFunctionService.java  | 20 +++++
 .../ortus/boxlang/runtime/scopes/Key.java     |  1 +
 .../exceptions/BoxValidationException.java    |  2 +-
 .../validation/dynamic/ValueOneOf.java        |  4 +-
 .../ortus/boxlang/compiler/QoQParseTest.java  | 71 ++++++++++++----
 6 files changed, 161 insertions(+), 19 deletions(-)
 create mode 100644 src/main/java/ortus/boxlang/runtime/bifs/global/query/QueryRegisterFunction.java

diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/query/QueryRegisterFunction.java b/src/main/java/ortus/boxlang/runtime/bifs/global/query/QueryRegisterFunction.java
new file mode 100644
index 000000000..44b4dd242
--- /dev/null
+++ b/src/main/java/ortus/boxlang/runtime/bifs/global/query/QueryRegisterFunction.java
@@ -0,0 +1,82 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package ortus.boxlang.runtime.bifs.global.query;
+
+import java.util.Set;
+
+import ortus.boxlang.runtime.bifs.BIF;
+import ortus.boxlang.runtime.bifs.BoxBIF;
+import ortus.boxlang.runtime.context.IBoxContext;
+import ortus.boxlang.runtime.jdbc.qoq.QoQFunctionService;
+import ortus.boxlang.runtime.scopes.ArgumentsScope;
+import ortus.boxlang.runtime.scopes.Key;
+import ortus.boxlang.runtime.types.Argument;
+import ortus.boxlang.runtime.types.Function;
+import ortus.boxlang.runtime.types.QueryColumnType;
+import ortus.boxlang.runtime.validation.Validator;
+
+@BoxBIF
+public class QueryRegisterFunction extends BIF {
+
+	/**
+	 * Constructor
+	 */
+	public QueryRegisterFunction() {
+		super();
+		declaredArguments = new Argument[] {
+		    new Argument( true, "string", Key._NAME ),
+		    new Argument( true, "function", Key.function ),
+		    new Argument( true, "string", Key.returnType, "Object" ),
+		    new Argument( true, "string", Key.type, "scalar", Set.of( Validator.valueOneOf( "scalar", "aggregate" ) ) )
+		};
+	}
+
+	/**
+	 * Register a new scalar or aggregate function for use with Query of Query. Functions only need to be registered once per runtime and they will be cached in memory until
+	 * the runtime restarts.
+	 *
+	 * @param context   The context in which the BIF is being invoked.
+	 * @param arguments Argument scope for the BIF.
+	 *
+	 * @argument.name The name of the function to register without parenthesis. You will reference it in your SQL with this name.
+	 * 
+	 * @argument.function The function to register. This can be a closure or a function reference. For scalar functions, the function will receive a value for each incoming argument and must return a single value.
+	 *                    For aggregate functions, the function will receive a value for each incoming argument in the form of an array of values representing each row being aggregated and must return a single value.
+	 *                    Null values are not passed into aggregates. Aggregate functions should return a scalar value
+	 *                    so the size of the array won't necessarily match the number of rows in the query, and may be different across columns.
+	 * 
+	 * @argument.type The type of function to register. This can be "scalar" or "aggregate". Default is "scalar".
+	 * 
+	 * @argument.returnType The return type of the function as a query column type. Default is "Object". This can be any valid query column type. This can be used to enforce
+	 *                      how the return values of this function are handled. For example, customFunc() + customFunc2() will behave differently based on whether the custom functions return strings or numbers.
+	 * 
+	 */
+	public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
+		Key				name		= Key.of( arguments.getAsString( Key._NAME ) );
+		Function		function	= arguments.getAsFunction( Key.function );
+		Key				type		= Key.of( arguments.getAsString( Key.type ) );
+		QueryColumnType	returnType	= QueryColumnType.fromString( arguments.getAsString( Key.returnType ) );
+
+		// Passing 0 for required args for now. It's a runtime check, so the UDF being passed can just declare it's args as required anyway.
+		if ( type.equals( Key.scalar ) ) {
+			QoQFunctionService.registerCustom( name, function, returnType, 0, context );
+		} else {
+			QoQFunctionService.registerCustomAggregate( name, function, returnType, 0, context );
+		}
+
+		return null;
+	}
+
+}
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
index 2ff2ad29b..0300c82e9 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
@@ -144,6 +144,26 @@ public static void registerCustom( Key name, ortus.boxlang.runtime.types.Functio
 		) );
 	}
 
+	/**
+	 * Register a custom aggregate function based on a UDF or closure
+	 * 
+	 * @param name           The name of the function
+	 * @param function       The function to execute
+	 * @param returnType     The return type of the function
+	 * @param requiredParams The number of required parameters
+	 * @param context        The context to execute the function in
+	 */
+	public static void registerCustomAggregate( Key name, ortus.boxlang.runtime.types.Function function, QueryColumnType returnType, int requiredParams,
+	    IBoxContext context ) {
+		functions.put( name, QoQFunction.ofAggregate(
+		    // TODO: do we pass the expressions here?
+		    ( List arguments, List expressions ) -> context.invokeFunction( function, arguments.toArray() ),
+		    null,
+		    returnType,
+		    requiredParams
+		) );
+	}
+
 	/**
 	 * Register an aggregate function via a BiFunction
 	 * 
diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
index 805fbb38a..11b170fa0 100644
--- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java
+++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
@@ -628,6 +628,7 @@ public class Key implements Comparable, Serializable {
 	public static final Key		runnable							= Key.of( "runnable" );
 	public static final Key		runtime								= Key.of( "runtime" );
 	public static final Key		samesite							= Key.of( "samesite" );
+	public static final Key		scalar								= Key.of( "scalar" );
 	public static final Key		scale								= Key.of( "scale" );
 	public static final Key		schedulerService					= Key.of( "schedulerService" );
 	public static final Key		scope								= Key.of( "scope" );
diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
index 2586df1d2..2f827190d 100644
--- a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
+++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
@@ -42,7 +42,7 @@ public BoxValidationException( String message ) {
 	 * @param message The message
 	 */
 	public BoxValidationException( Key caller, Validatable record, String message ) {
-		this( "Input [" + record.name().getName() + "] for component [" + caller.getName() + "] " + message, null );
+		this( "Input [" + record.name().getName() + "] for [" + caller.getName() + "] " + message, null );
 	}
 
 	/**
diff --git a/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java b/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
index 3c8a6f3bc..eef10f6a4 100644
--- a/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
+++ b/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
@@ -39,8 +39,8 @@ public ValueOneOf( Set validValues ) {
 
 	public void validate( IBoxContext context, Key caller, Validatable record, IStruct records ) {
 		if ( records.containsKey( record.name() ) ) {
-			String value = records.getAsString( record.name() );
-			if ( value != null && !validValues.stream().map( String::toLowerCase ).anyMatch( value.toLowerCase()::equals ) ) {
+			String value = records.getAsString( record.name() ).toLowerCase();
+			if ( value != null && !validValues.stream().map( String::toLowerCase ).anyMatch( value::equals ) ) {
 				throw new BoxValidationException( caller, record, "must be one of the following values: " + String.join( ", ", validValues ) );
 			}
 		}
diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
index 0ff7236ce..a42227b38 100644
--- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
+++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
@@ -255,22 +255,61 @@ where name not in ( select name from qryMen )
 	public void testcustomFunc() {
 		instance.executeSource(
 		    """
-		       import ortus.boxlang.runtime.jdbc.qoq.QoQFunctionService;
-		       import ortus.boxlang.runtime.scopes.Key;
-		       import ortus.boxlang.runtime.types.QueryColumnType;
-
-		    // Register a custom function
-		       QoQFunctionService.registerCustom( Key.of("reverse"), ::reverse, QueryColumnType.VARCHAR, 1, getBoxContext() );
-
-		                           q = queryExecute( "
-		                  select reverse( 'Brad' ) as rev
-		                                  ",
-		                                        	[],
-		                                        	{ dbType : "query" }
-		                                        );
-		                                     println( q )
-
-		                                        """,
+		       // Register a custom function
+		    queryRegisterFunction( "reverse", ::reverse, "varchar" )
+
+		                              q = queryExecute( "
+		                     select reverse( 'Brad' ) as rev
+		                                     ",
+		                                           	[],
+		                                           	{ dbType : "query" }
+		                                           );
+		                                        println( q )
+
+		                                           """,
+		    context );
+	}
+
+	@Test
+	public void testcustomFuncAggregate() {
+		instance.executeSource(
+		    """
+		       // Register a custom aggregate function
+		    queryRegisterFunction(  "arrayToList", ::arrayToList, "varchar", "aggregate" )
+
+		       qryAll = queryNew( "name", "varchar", [["Luis"],["Jon"],["Brad"],["Esme"],["Myrna"]] )
+		                              q = queryExecute( "
+		                     select arrayToList( name ) as names
+		         from qryAll
+		                                     ",
+		                                           	[],
+		                                           	{ dbType : "query" }
+		                                           );
+		                                        println( q )
+
+		                                           """,
+		    context );
+	}
+
+	@Test
+	public void testcustomFuncAggregateUDF() {
+		instance.executeSource(
+		    """
+		         // Register a custom aggregate function
+		    queryRegisterFunction(  "fullNameList", (firsts,lasts)->firsts.reduce( (acc,first,i)=>acc.listAppend( first & ' ' & lasts[i] ), "" ), "varchar", "aggregate" )
+
+		         qryAll = queryNew( "first,last", "varchar,varchar", [["Luis","Majano"],["Jon","Clausen"],["Brad","Wood"],["Esme","Acevedo"],["Myrna","Nelly"]] )
+		                                q = queryExecute( "
+		                       select fullNameList( first, last ) as fullNames,
+		       group_concat( first + ' ' + last ) as fullNames2
+		       from qryAll
+		                                       ",
+		                                             	[],
+		                                             	{ dbType : "query" }
+		                                             );
+		                                          println( q )
+
+		                                             """,
 		    context );
 	}
 

From e08339367e6fe9d87fdb9a043949244d54574735 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Mon, 13 Jan 2025 20:31:00 -0600
Subject: [PATCH 129/161] BL-933

---
 .../ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java b/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
index eef10f6a4..b1b184bd1 100644
--- a/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
+++ b/src/main/java/ortus/boxlang/runtime/validation/dynamic/ValueOneOf.java
@@ -39,7 +39,10 @@ public ValueOneOf( Set validValues ) {
 
 	public void validate( IBoxContext context, Key caller, Validatable record, IStruct records ) {
 		if ( records.containsKey( record.name() ) ) {
-			String value = records.getAsString( record.name() ).toLowerCase();
+			String value = records.getAsString( record.name() );
+			if ( value != null ) {
+				value = value.toLowerCase();
+			}
 			if ( value != null && !validValues.stream().map( String::toLowerCase ).anyMatch( value::equals ) ) {
 				throw new BoxValidationException( caller, record, "must be one of the following values: " + String.join( ", ", validValues ) );
 			}

From ec769d7eaa3a712972268ff2f6ad56fa6d074fd3 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Mon, 13 Jan 2025 23:24:41 -0600
Subject: [PATCH 130/161] meh, a bunch of tests rely on this wording

---
 .../runtime/types/exceptions/BoxValidationException.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
index 2f827190d..2586df1d2 100644
--- a/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
+++ b/src/main/java/ortus/boxlang/runtime/types/exceptions/BoxValidationException.java
@@ -42,7 +42,7 @@ public BoxValidationException( String message ) {
 	 * @param message The message
 	 */
 	public BoxValidationException( Key caller, Validatable record, String message ) {
-		this( "Input [" + record.name().getName() + "] for [" + caller.getName() + "] " + message, null );
+		this( "Input [" + record.name().getName() + "] for component [" + caller.getName() + "] " + message, null );
 	}
 
 	/**

From 0659afc1d77e38367ab1f4e664e3ceb1bbbed7d6 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Mon, 13 Jan 2025 23:26:37 -0600
Subject: [PATCH 131/161] Final cleanup BL-933

---
 .../runtime/jdbc/qoq/LikeOperation.java       |  3 +-
 .../runtime/jdbc/qoq/QoQFunctionService.java  |  4 +--
 .../jdbc/qoq/functions/scalar/Length.java     |  3 ++
 .../jdbc/qoq/functions/scalar/Lower.java      |  3 ++
 .../jdbc/qoq/functions/scalar/Trim.java       |  3 ++
 .../jdbc/qoq/functions/scalar/Upper.java      |  3 ++
 .../ortus/boxlang/compiler/QoQParseTest.java  | 28 ++++++-------------
 7 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/LikeOperation.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/LikeOperation.java
index 69d825d39..71993caf6 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/LikeOperation.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/LikeOperation.java
@@ -77,7 +77,7 @@ private static void escapeForRegex( StringBuilder sb, char c ) {
 	 * Create a pattern from the patternToSearchFor
 	 */
 	private static Pattern createPattern( String patternToSearchFor, String escape ) {
-		var	patternCacheKey	= patternToSearchFor + escape == null ? "" : escape;
+		var	patternCacheKey	= patternToSearchFor + ( escape == null ? "" : escape );
 		var	pattern			= patterns.get( patternCacheKey );
 		if ( pattern != null )
 			return pattern;
@@ -130,6 +130,7 @@ else if ( c == '[' ) {
 
 			}
 			try {
+				System.out.println( "compiling LIKE Pattern: " + sb.toString() );
 				patterns.put( patternCacheKey, pattern = Pattern.compile( sb.toString(), Pattern.DOTALL ) );
 			} catch ( PatternSyntaxException e ) {
 				throw new DatabaseException( "Invalid LIKE pattern [" + patternToSearchFor + "] has been specified in a LIKE conditional", e );
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
index 0300c82e9..ef0a109aa 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/QoQFunctionService.java
@@ -95,12 +95,12 @@ public class QoQFunctionService {
 		register( Upper.INSTANCE );
 		register( Left.INSTANCE );
 		register( Right.INSTANCE );
+		register( Cast.INSTANCE );
+		register( Convert.INSTANCE );
 
 		// Aggregate
 		register( Max.INSTANCE );
 		register( Min.INSTANCE );
-		register( Cast.INSTANCE );
-		register( Convert.INSTANCE );
 		register( Sum.INSTANCE );
 		register( Avg.INSTANCE );
 		register( GroupConcat.INSTANCE );
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java
index f958e090d..5646d083e 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Length.java
@@ -46,6 +46,9 @@ public int getMinArgs() {
 
 	@Override
 	public Object apply( List args, List expressions ) {
+		if ( args.get( 0 ) == null ) {
+			return 0;
+		}
 		return StringCaster.cast( args.get( 0 ) ).length();
 	}
 
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java
index 180e09f48..a1f1385d5 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Lower.java
@@ -46,6 +46,9 @@ public int getMinArgs() {
 
 	@Override
 	public Object apply( List args, List expressions ) {
+		if ( args.get( 0 ) == null ) {
+			return null;
+		}
 		return StringCaster.cast( args.get( 0 ) ).toLowerCase();
 	}
 
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java
index 100602ed9..6aaf14b01 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Trim.java
@@ -46,6 +46,9 @@ public int getMinArgs() {
 
 	@Override
 	public Object apply( List args, List expressions ) {
+		if ( args.get( 0 ) == null ) {
+			return null;
+		}
 		return StringCaster.cast( args.get( 0 ) ).trim();
 	}
 
diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java
index 5fcab6449..9d912833c 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/qoq/functions/scalar/Upper.java
@@ -46,6 +46,9 @@ public int getMinArgs() {
 
 	@Override
 	public Object apply( List args, List expressions ) {
+		if ( args.get( 0 ) == null ) {
+			return null;
+		}
 		return StringCaster.cast( args.get( 0 ) ).toUpperCase();
 	}
 
diff --git a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
index a42227b38..9a51cd701 100644
--- a/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
+++ b/src/test/java/ortus/boxlang/compiler/QoQParseTest.java
@@ -604,26 +604,14 @@ public void testNullAggregate() {
 	public void testsdf() {
 		instance.executeSource(
 		    """
-		       queryWithDataIn = QueryNew('id,value', 'integer,varchar',[[1,'a'],[2,'b'],[3,'c'],[4,'d'],[5,'e']]);
-		     actual = QueryExecute(
-		     				params = [
-		     					{ value: 3  },
-		     					{ value: '3,4' , sqltype: 'numeric' , list = true }
-		     				],
-		     				options = {
-		     					dbtype: 'query'
-		     				},
-		     				sql = "
-		     					SELECT
-		     						id,
-		     						value
-		     					FROM queryWithDataIn
-		     					WHERE id = ?
-		     		and id IN ( ? )
-		    "	);
-
-		     			println(actual)
-		                   			                          """,
+		        q = QueryExecute("
+		        				select 1 ^ 2
+		       ",
+		    [],
+		    { dbType : "query" } );
+
+		        			println(q)
+		                      			                          """,
 		    context, BoxSourceType.CFSCRIPT );
 	}
 

From b1a1af2c5e0710adcdf3ad9a8433f2cfc62841cd Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 15:41:08 +0100
Subject: [PATCH 132/161] BL-936 #resolve When doing named parameter queries
 and you send more parameters than required, it should ignore them, not throw
 an exception that you sent more

---
 .../boxlang/runtime/jdbc/PendingQuery.java      | 11 +++--------
 .../boxlang/runtime/net/HTTPStatusReasons.java  | 17 +++++++++++++++++
 .../ortus/boxlang/runtime/net/HttpManager.java  | 17 +++++++++++++++++
 .../runtime/net/HttpRequestMultipartBody.java   | 17 +++++++++++++++++
 .../boxlang/runtime/net/NameValuePair.java      | 17 +++++++++++++++++
 .../ortus/boxlang/runtime/net/URIBuilder.java   | 17 +++++++++++++++++
 6 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java
index 6e1ba0e02..bf23fdb1a 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/PendingQuery.java
@@ -461,11 +461,6 @@ private List buildParameterList( Array positionalParameters, ISt
 		if ( isPositional && positionalParameters.size() > paramsEncountered ) {
 			throw new DatabaseException( "Too many positional parameters [" + positionalParameters.size()
 			    + "] provided for query having only [" + paramsEncountered + "] '?' char(s)." );
-		} else if ( !isPositional && namedParameters.keySet().size() != foundNamedParams.size() ) {
-			// Make sure all named params were used
-			Set missingParams = new HashSet<>( namedParameters.keySet() );
-			missingParams.removeAll( foundNamedParams );
-			throw new DatabaseException( "Named parameter(s) [" + missingParams + "] provided to query were not used." );
 		}
 
 		SQLWithParamTokens.add( SQLWithParamToken.toString() );
@@ -484,7 +479,7 @@ private List buildParameterList( Array positionalParameters, ISt
 
 	/**
 	 * Return final SQL with paramter values
-	 * 
+	 *
 	 * @return
 	 */
 	public @Nonnull String getSQLWithParamValues() {
@@ -662,7 +657,7 @@ private ExecutedQuery respondWithCachedQuery( ExecutedQuery cachedQuery ) {
 	 * If this is a paramaterized query, apply the parameters to the provided statement.
 	 * We will also take this opportunity to finalize the list of SQL tokens with the
 	 * final param values to build the effective SQL string.
-	 * 
+	 *
 	 * @throws SQLException
 	 */
 	private void applyParameters( Statement statement, IBoxContext context ) throws SQLException {
@@ -717,7 +712,7 @@ private void applyParameters( Statement statement, IBoxContext context ) throws
 
 	/**
 	 * Emit a value to the SQL string, quoting it if necessary.
-	 * 
+	 *
 	 * @param SQLWithParamValues The SQL string to append the value to.
 	 * @param value              The value to append.
 	 * @param type               The type of the value.
diff --git a/src/main/java/ortus/boxlang/runtime/net/HTTPStatusReasons.java b/src/main/java/ortus/boxlang/runtime/net/HTTPStatusReasons.java
index 6aa6da485..9372af0d7 100644
--- a/src/main/java/ortus/boxlang/runtime/net/HTTPStatusReasons.java
+++ b/src/main/java/ortus/boxlang/runtime/net/HTTPStatusReasons.java
@@ -1,3 +1,20 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ortus.boxlang.runtime.net;
 
 import java.util.Map;
diff --git a/src/main/java/ortus/boxlang/runtime/net/HttpManager.java b/src/main/java/ortus/boxlang/runtime/net/HttpManager.java
index 1e2206321..6fd0a412c 100644
--- a/src/main/java/ortus/boxlang/runtime/net/HttpManager.java
+++ b/src/main/java/ortus/boxlang/runtime/net/HttpManager.java
@@ -1,3 +1,20 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ortus.boxlang.runtime.net;
 
 import java.net.URI;
diff --git a/src/main/java/ortus/boxlang/runtime/net/HttpRequestMultipartBody.java b/src/main/java/ortus/boxlang/runtime/net/HttpRequestMultipartBody.java
index 252b4fa92..ce4c7a5f3 100644
--- a/src/main/java/ortus/boxlang/runtime/net/HttpRequestMultipartBody.java
+++ b/src/main/java/ortus/boxlang/runtime/net/HttpRequestMultipartBody.java
@@ -1,3 +1,20 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ortus.boxlang.runtime.net;
 
 import java.io.ByteArrayOutputStream;
diff --git a/src/main/java/ortus/boxlang/runtime/net/NameValuePair.java b/src/main/java/ortus/boxlang/runtime/net/NameValuePair.java
index 9b2948a81..05d9ea097 100644
--- a/src/main/java/ortus/boxlang/runtime/net/NameValuePair.java
+++ b/src/main/java/ortus/boxlang/runtime/net/NameValuePair.java
@@ -1,3 +1,20 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ortus.boxlang.runtime.net;
 
 import javax.annotation.Nonnull;
diff --git a/src/main/java/ortus/boxlang/runtime/net/URIBuilder.java b/src/main/java/ortus/boxlang/runtime/net/URIBuilder.java
index d0b282852..50d1b3b62 100644
--- a/src/main/java/ortus/boxlang/runtime/net/URIBuilder.java
+++ b/src/main/java/ortus/boxlang/runtime/net/URIBuilder.java
@@ -1,3 +1,20 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ortus.boxlang.runtime.net;
 
 import ortus.boxlang.runtime.net.NameValuePair;

From f7ff781a1e6246ba18b27023321b0d8906fef994 Mon Sep 17 00:00:00 2001
From: Jacob Beers 
Date: Tue, 14 Jan 2025 08:52:02 -0600
Subject: [PATCH 133/161] BL-927 Add test to check multiple assignment

---
 src/test/java/TestCases/phase1/AssignmentClass.bx | 10 ++++++++++
 .../java/TestCases/phase1/AssignmentTest.java     | 15 +++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 src/test/java/TestCases/phase1/AssignmentClass.bx

diff --git a/src/test/java/TestCases/phase1/AssignmentClass.bx b/src/test/java/TestCases/phase1/AssignmentClass.bx
new file mode 100644
index 000000000..89f28fa27
--- /dev/null
+++ b/src/test/java/TestCases/phase1/AssignmentClass.bx
@@ -0,0 +1,10 @@
+class {
+    
+    function init(){
+        variables.a = this.a = "test"
+    }
+
+    function getA(){
+        return variables.a;
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/TestCases/phase1/AssignmentTest.java b/src/test/java/TestCases/phase1/AssignmentTest.java
index c4ae5b5e5..babb62f5d 100644
--- a/src/test/java/TestCases/phase1/AssignmentTest.java
+++ b/src/test/java/TestCases/phase1/AssignmentTest.java
@@ -217,4 +217,19 @@ public void testQuotedAssignment2() {
 
 	}
 
+	@DisplayName( "multiple assignment" )
+	@Test
+	public void testMultipleAssignment() {
+		instance.executeSource(
+		    """
+		       thing = new src.test.java.TestCases.phase1.AssignmentClass();
+		    vara = thing.getA();
+		    thisA = thing.a;
+		               """,
+		    context );
+		assertThat( variables.get( "varA" ) ).isEqualTo( "test" );
+		assertThat( variables.get( "thisA" ) ).isEqualTo( "test" );
+
+	}
+
 }

From 3739e95705c6bfbf5f9ce4c130a6209e8bf649c4 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 17:54:07 +0100
Subject: [PATCH 134/161] fixes tests with negative hashcodes

---
 .../ortus/boxlang/runtime/config/segments/DatasourceConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java
index c81345d0c..d0e8cf31e 100644
--- a/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java
+++ b/src/main/java/ortus/boxlang/runtime/config/segments/DatasourceConfig.java
@@ -333,7 +333,7 @@ public Key getUniqueName() {
 		uniqueName.append( "_" );
 
 		// Hash the properties
-		uniqueName.append( properties.hashCode() );
+		uniqueName.append( Math.abs( properties.hashCode() ) );
 
 		return Key.of( uniqueName.toString() );
 	}

From 0321afa6935d90cd642bda734e6187b0cace3d22 Mon Sep 17 00:00:00 2001
From: Jacob Beers 
Date: Tue, 14 Jan 2025 11:59:03 -0600
Subject: [PATCH 135/161] BL-926 Fix do while in psuedo constructor

---
 .../transformer/expression/BoxBreakTransformer.java       | 8 +-------
 src/test/java/TestCases/phase3/ClassTest.java             | 2 +-
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBreakTransformer.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBreakTransformer.java
index 4ba934e57..89111e785 100644
--- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBreakTransformer.java
+++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/transformer/expression/BoxBreakTransformer.java
@@ -90,18 +90,12 @@ public List transform( BoxNode node, TransformerContext contex
 			nodes.add( new InsnNode( Opcodes.ARETURN ) );
 			return AsmHelper.addLineNumberLabels( nodes, node );
 		} else if ( exitsAllowed.equals( ExitsAllowed.LOOP ) ) {
-			// template = "if(true) break " + breakLabel + ";";
-			nodes.add( new InsnNode( Opcodes.ARETURN ) );
+			nodes.add( new InsnNode( transpiler.canReturn() ? Opcodes.ARETURN : Opcodes.RETURN ) );
 			return AsmHelper.addLineNumberLabels( nodes, node );
 		} else if ( exitsAllowed.equals( ExitsAllowed.FUNCTION ) ) {
 			nodes.add( new InsnNode( Opcodes.ARETURN ) );
 			return AsmHelper.addLineNumberLabels( nodes, node );
-		} else {
-			// template = "if(true) return;";
 		}
-		// if ( currentBreak == null ) {
-		// throw new RuntimeException( "Cannot break from current location" );
-		// }
 
 		throw new RuntimeException( "Cannot break from current location" );
 
diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java
index bb8a28522..9e877bb38 100644
--- a/src/test/java/TestCases/phase3/ClassTest.java
+++ b/src/test/java/TestCases/phase3/ClassTest.java
@@ -1646,10 +1646,10 @@ public void testPropertiesNotInheritedInMetadata() {
 	}
 
 	@Test
-	@Disabled( "BL-926" )
 	public void testUserASMError() {
 		instance.executeSource(
 		    """
+
 		    new src.test.java.TestCases.phase3.ASMError()
 		      """,
 		    context );

From a50ec270ee51ba9886786aaf912b49a0c7e64b72 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Tue, 14 Jan 2025 13:27:58 -0600
Subject: [PATCH 136/161] BL-764

---
 .../runtime/bifs/global/xml/XMLSearch.java    | 60 ++++++++++++-------
 .../bifs/global/xml/XMLSearchTest.java        | 42 +++++++++++++
 2 files changed, 82 insertions(+), 20 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearch.java b/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearch.java
index 3a359b15f..593241273 100644
--- a/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearch.java
+++ b/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearch.java
@@ -28,6 +28,8 @@
 import ortus.boxlang.runtime.bifs.BoxBIF;
 import ortus.boxlang.runtime.bifs.BoxMember;
 import ortus.boxlang.runtime.context.IBoxContext;
+import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
+import ortus.boxlang.runtime.dynamic.casters.NumberCaster;
 import ortus.boxlang.runtime.scopes.ArgumentsScope;
 import ortus.boxlang.runtime.scopes.Key;
 import ortus.boxlang.runtime.types.Argument;
@@ -68,30 +70,35 @@ public XMLSearch() {
 	 *
 	 */
 	public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
-		XML				xml			= arguments.getAsXML( Key.XMLNode );
-		String			xpathString	= arguments.getAsString( Key.xpath );
-		final IStruct	params		= arguments.getAsStruct( Key.params );
-		try {
+		XML				xml				= arguments.getAsXML( Key.XMLNode );
+		String			xpathString		= arguments.getAsString( Key.xpath );
+		final IStruct	params			= arguments.getAsStruct( Key.params );
 
-			// Create an XPathFactory
-			XPathFactory	xPathFactory	= XPathFactory.newInstance();
+		// Create an XPathFactory
+		XPathFactory	xPathFactory	= XPathFactory.newInstance();
 
-			// Create an XPath object
-			XPath			xpath			= xPathFactory.newXPath();
+		// Create an XPath object
+		XPath			xpath			= xPathFactory.newXPath();
 
-			xpath.setXPathVariableResolver( new XPathVariableResolver() {
+		xpath.setXPathVariableResolver( new XPathVariableResolver() {
 
-				public Object resolveVariable( QName variableName ) {
-					return params.get( Key.of( variableName.getLocalPart() ) );
-				}
-			} );
+			public Object resolveVariable( QName variableName ) {
+				return params.get( Key.of( variableName.getLocalPart() ) );
+			}
+		} );
 
+		XPathExpression expression;
+		try {
 			// TODO: cache compiled expressions
-			XPathExpression	expression	= xpath.compile( xpathString );
+			expression = xpath.compile( xpathString );
+		} catch ( XPathExpressionException e ) {
+			throw new BoxRuntimeException( "Error compiling XPath: " + xpathString, e );
+		}
 
+		try {
 			// Evaluate the XPath expression on the Document
-			Object			result		= expression.evaluate( xml.getNode(), XPathConstants.NODESET );
-			Array			results		= new Array();
+			Object	result	= expression.evaluate( xml.getNode(), XPathConstants.NODESET );
+			Array	results	= new Array();
 			// Process the result
 			if ( result instanceof NodeList nodeList ) {
 				for ( int i = 0; i < nodeList.getLength(); i++ ) {
@@ -100,10 +107,23 @@ public Object resolveVariable( QName variableName ) {
 			}
 			return results;
 
-		} catch (
-
-		XPathExpressionException e ) {
-			throw new BoxRuntimeException( "Invalid XPath: " + xpathString, e );
+		} catch ( XPathExpressionException e ) {
+			// The API here is freaking worthless. It's impossible to tell what kind of return type you'll get without doing your own manual pre-parsing of the xpath string.
+			// So, we have to just try it as a nodeset and if that fails, guess what it should have been by analyzing the error message. Pathetic.
+			String message = e.getMessage() == null ? "" : e.getMessage();
+			try {
+				if ( message.indexOf( "#BOOLEAN" ) != -1 ) {
+					return BooleanCaster.cast( expression.evaluate( xml.getNode(), XPathConstants.BOOLEAN ) );
+				} else if ( message.indexOf( "#NUMBER" ) != -1 ) {
+					return NumberCaster.cast( expression.evaluate( xml.getNode(), XPathConstants.NUMBER ) );
+				} else if ( message.indexOf( "#STRING" ) != -1 ) {
+					return expression.evaluate( xml.getNode(), XPathConstants.STRING );
+				} else {
+					throw e;
+				}
+			} catch ( XPathExpressionException e1 ) {
+				throw new BoxRuntimeException( "Error evaluating XPath: " + xpathString, e1 );
+			}
 		}
 	}
 
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearchTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearchTest.java
index 638a47f03..346e143f5 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearchTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLSearchTest.java
@@ -111,4 +111,46 @@ public void testCanSearchMember() {
 		    .isEqualTo( "luis" );
 	}
 
+	@DisplayName( "It can search string" )
+	@Test
+	public void testCanSearchString() {
+		instance.executeSource(
+		    """
+		    xml = '';
+
+		    result = XmlSearch(xml, "string(/Conditions/@NotBefore)");
+		         """,
+		    context );
+		assertThat( variables.get( result ) ).isInstanceOf( String.class );
+		assertThat( variables.get( result ) ).isEqualTo( "2018-08-24T10:54:19.464Z" );
+	}
+
+	@DisplayName( "It can search boolean" )
+	@Test
+	public void testCanSearchBoolean() {
+		instance.executeSource(
+		    """
+		    xml = '';
+
+		    result = XmlSearch(xml, "boolean(/Conditions/@IsActive)");
+		    """,
+		    context );
+		assertThat( variables.get( result ) ).isInstanceOf( Boolean.class );
+		assertThat( variables.get( result ) ).isEqualTo( true );
+	}
+
+	@DisplayName( "It can search numeric" )
+	@Test
+	public void testCanSearchNumeric() {
+		instance.executeSource(
+		    """
+		    xml = '';
+
+		    result = XmlSearch(xml, "number(/Item/@Price)");
+		    """,
+		    context );
+		assertThat( variables.get( result ) ).isInstanceOf( Double.class );
+		assertThat( variables.get( result ) ).isEqualTo( 19.99 );
+	}
+
 }

From 50565424f62c3a1b9ca7a32a67526ab68f4e1e54 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 19:00:17 +0100
Subject: [PATCH 137/161] BL-937 #resolve getOrCreateSession() relying on the
 starting listener for settings, when it should look at the context to reflect
 changes if any

---
 .../runtime/application/Application.java      | 38 +++++++++++--------
 .../application/BaseApplicationListener.java  |  4 +-
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/application/Application.java b/src/main/java/ortus/boxlang/runtime/application/Application.java
index 9f6637f99..77c4bd126 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Application.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Application.java
@@ -35,7 +35,6 @@
 import ortus.boxlang.runtime.context.ApplicationBoxContext;
 import ortus.boxlang.runtime.context.IBoxContext;
 import ortus.boxlang.runtime.context.RequestBoxContext;
-import ortus.boxlang.runtime.context.ScriptingRequestBoxContext;
 import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
 import ortus.boxlang.runtime.dynamic.casters.LongCaster;
 import ortus.boxlang.runtime.dynamic.casters.StringCaster;
@@ -371,13 +370,14 @@ private void startupSessionStorage( ApplicationBoxContext appContext ) {
 	/**
 	 * Get a session by ID for this application, creating if neccessary if not found
 	 *
-	 * @param ID The ID of the session
+	 * @param ID      The ID of the session
+	 * @param context The context of the request that is creating/getting the session
 	 *
 	 * @return The session object
 	 */
-	public Session getOrCreateSession( Key ID ) {
+	public Session getOrCreateSession( Key ID, RequestBoxContext context ) {
 		Duration	timeoutDuration	= null;
-		Object		sessionTimeout	= this.startingListener.getSettings().get( Key.sessionTimeout );
+		Object		sessionTimeout	= context.getConfigItems( Key.applicationSettings, Key.sessionTimeout );
 
 		// Duration is the default, but if not, we will use the number as seconds
 		// Which is what the cache providers expect
@@ -507,17 +507,21 @@ public synchronized void shutdown( boolean force ) {
 
 		// Announce it globally
 		RequestBoxContext requestContext = this.getStartingListener().getRequestContext();
-		BoxRuntime.getInstance().getInterceptorService().announce( Key.onApplicationEnd, Struct.of(
-		    "application", this,
-		    "context", requestContext
-		) );
+		try {
+			BoxRuntime.getInstance().getInterceptorService().announce( Key.onApplicationEnd, Struct.of(
+			    "application", this,
+			    "context", requestContext
+			) );
+		} catch ( Exception e ) {
+			logger.error( "Error announcing onApplicationEnd", e );
+		}
 
-		// Shutdown all sessions
+		// Shutdown all sessions if NOT in a cluster
 		if ( !BooleanCaster.cast( this.startingListener.getSettings().get( Key.sessionCluster ) ) ) {
 			sessionsCache.getKeysStream( sessionCacheFilter )
 			    .parallel()
 			    .map( Key::of )
-			    .map( this::getOrCreateSession )
+			    .map( sessionKey -> ( Session ) sessionsCache.get( sessionKey.getName() ).get() )
 			    .forEach( session -> session.shutdown( this.getStartingListener() ) );
 		}
 
@@ -532,11 +536,15 @@ public synchronized void shutdown( boolean force ) {
 
 		// Announce it to the listener
 		if ( this.startingListener != null ) {
-			// Any buffer output in this context will be discarded
-			this.startingListener.onApplicationEnd(
-			    requestContext,
-			    new Object[] { applicationScope }
-			);
+			try {
+				// Any buffer output in this context will be discarded
+				this.startingListener.onApplicationEnd(
+				    requestContext,
+				    new Object[] { applicationScope }
+				);
+			} catch ( Exception e ) {
+				logger.error( "Error calling onApplicationEnd", e );
+			}
 		}
 
 		// Clear out the data
diff --git a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java
index 0af7b5727..be657f4cc 100644
--- a/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java
+++ b/src/main/java/ortus/boxlang/runtime/application/BaseApplicationListener.java
@@ -353,7 +353,7 @@ private void createOrUpdateSessionManagement() {
 		else {
 			if ( sessionManagementEnabled ) {
 				// Ensure we have the right session (app name could have changed)
-				existingSessionContext.updateSession( this.application.getOrCreateSession( this.context.getSessionID() ) );
+				existingSessionContext.updateSession( this.application.getOrCreateSession( this.context.getSessionID(), this.context ) );
 				// Only starts the first time
 				existingSessionContext.getSession().start( this.context );
 			} else {
@@ -452,7 +452,7 @@ public void initializeSession( Key newID ) {
 		Session targetSession = this.context
 		    .getApplicationContext()
 		    .getApplication()
-		    .getOrCreateSession( newID );
+		    .getOrCreateSession( newID, this.context );
 		this.context.removeParentContext( SessionBoxContext.class );
 		this.context.injectTopParentContext( new SessionBoxContext( targetSession ) );
 		targetSession.start( this.context );

From ee5e31d5d80bc7393ad53edee254f738e8ce57de Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 19:21:29 +0100
Subject: [PATCH 138/161] small typo fix

---
 .../java/ortus/boxlang/runtime/application/Application.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/application/Application.java b/src/main/java/ortus/boxlang/runtime/application/Application.java
index 77c4bd126..c764b2e02 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Application.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Application.java
@@ -518,10 +518,10 @@ public synchronized void shutdown( boolean force ) {
 
 		// Shutdown all sessions if NOT in a cluster
 		if ( !BooleanCaster.cast( this.startingListener.getSettings().get( Key.sessionCluster ) ) ) {
-			sessionsCache.getKeysStream( sessionCacheFilter )
+			this.sessionsCache.getKeysStream( sessionCacheFilter )
 			    .parallel()
 			    .map( Key::of )
-			    .map( sessionKey -> ( Session ) sessionsCache.get( sessionKey.getName() ).get() )
+			    .map( sessionKey -> ( Session ) this.sessionsCache.get( sessionKey.getName() ).get() )
 			    .forEach( session -> session.shutdown( this.getStartingListener() ) );
 		}
 

From cc022bc22cedb1ee9f5e0161cdc2880f06567ca2 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 22:28:01 +0100
Subject: [PATCH 139/161] BL-914 #resolve Null session scope when attempting to
 update the last visit

---
 .../runtime/application/Application.java      | 18 +++++++++++--
 .../boxlang/runtime/application/Session.java  | 26 ++++++++++++++-----
 .../ortus/boxlang/runtime/scopes/Key.java     |  1 +
 3 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/application/Application.java b/src/main/java/ortus/boxlang/runtime/application/Application.java
index c764b2e02..9603bbfa0 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Application.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Application.java
@@ -362,6 +362,8 @@ private void startupSessionStorage( ApplicationBoxContext appContext ) {
 			        .get( key )
 			        .ifPresent( session -> ( ( Session ) session ).shutdown( this.startingListener ) );
 
+			    logger.debug( "Session storage cache [{}] shutdown and removed session [{}]", targetCache.getName(), key );
+
 			    return false;
 		    }, BoxEvent.BEFORE_CACHE_ELEMENT_REMOVED.key() );
 		logger.debug( "Session storage cache [{}] created for the application [{}]", sessionCacheName, this.name );
@@ -378,6 +380,7 @@ private void startupSessionStorage( ApplicationBoxContext appContext ) {
 	public Session getOrCreateSession( Key ID, RequestBoxContext context ) {
 		Duration	timeoutDuration	= null;
 		Object		sessionTimeout	= context.getConfigItems( Key.applicationSettings, Key.sessionTimeout );
+		String		cacheKey		= Session.buildCacheKey( ID, this.name );
 
 		// Duration is the default, but if not, we will use the number as seconds
 		// Which is what the cache providers expect
@@ -390,12 +393,23 @@ public Session getOrCreateSession( Key ID, RequestBoxContext context ) {
 		// logger.debug( "**** getOrCreateSession {} Timeout {} ", ID, timeoutDuration );
 
 		// Get or create the session
-		return ( Session ) this.sessionsCache.getOrSet(
-		    Session.buildCacheKey( ID, this.name ),
+		Session targetSession = ( Session ) this.sessionsCache.getOrSet(
+		    cacheKey,
 		    () -> new Session( ID, this ),
 		    timeoutDuration,
 		    timeoutDuration
 		);
+
+		// Is the session still valid?
+		if ( targetSession.isShutdown() ) {
+			// If not, remove it
+			this.sessionsCache.clear( cacheKey );
+			// And create a new one
+			targetSession = new Session( ID, this );
+			this.sessionsCache.set( cacheKey, targetSession, timeoutDuration, timeoutDuration );
+		}
+
+		return targetSession;
 	}
 
 	/**
diff --git a/src/main/java/ortus/boxlang/runtime/application/Session.java b/src/main/java/ortus/boxlang/runtime/application/Session.java
index 8ded0fca1..116a3a44a 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Session.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Session.java
@@ -63,6 +63,11 @@ public class Session implements Serializable {
 	 */
 	private boolean				isNew			= true;
 
+	/**
+	 * Flag for when session has been shutdown
+	 */
+	private boolean				isShutdown		= false;
+
 	/**
 	 * The application name linked to
 	 */
@@ -244,21 +249,30 @@ public void shutdown( BaseApplicationListener listener ) {
 			if ( this.sessionScope != null ) {
 				this.sessionScope.clear();
 			}
-			this.sessionScope = null;
+			this.sessionScope	= null;
+			this.isNew			= true;
+			this.isShutdown		= true;
 		}
 
 	}
 
+	/**
+	 * Tests if the session is still active or shutdown
+	 */
+	public boolean isShutdown() {
+		return this.isShutdown;
+	}
+
 	/**
 	 * Convert to string
 	 */
 	@Override
 	public String toString() {
 		return "Session{" +
-		    "ID=" + ID +
-		    ", sessionScope=" + sessionScope +
-		    ", isNew=" + isNew +
-		    ", applicationName=" + applicationName +
+		    "ID=" + this.ID +
+		    ", sessionScope=" + this.sessionScope.toString() +
+		    ", isNew=" + this.isNew +
+		    ", applicationName=" + this.applicationName +
 		    '}';
 	}
 
@@ -269,7 +283,7 @@ public IStruct asStruct() {
 		return Struct.of(
 		    Key.id, this.ID,
 		    Key.scope, this.sessionScope,
-		    "isNew", isNew,
+		    Key.isNew, this.isNew,
 		    Key.applicationName, this.applicationName
 		);
 	}
diff --git a/src/main/java/ortus/boxlang/runtime/scopes/Key.java b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
index 11b170fa0..95691ab43 100644
--- a/src/main/java/ortus/boxlang/runtime/scopes/Key.java
+++ b/src/main/java/ortus/boxlang/runtime/scopes/Key.java
@@ -399,6 +399,7 @@ public class Key implements Comparable, Serializable {
 	public static final Key		interfaces							= Key.of( "interfaces" );
 	public static final Key		interrupted							= Key.of( "interrupted" );
 	public static final Key		interval							= Key.of( "interval" );
+	public static final Key		isNew								= Key.of( "isNew" );
 	public static final Key		invoke								= Key.of( "invoke" );
 	public static final Key		invokeArgs							= Key.of( "invokeArgs" );
 	public static final Key		invokeImplicitAccessor				= Key.of( "invokeImplicitAccessor" );

From 45f913809a48b7ec732a81884fe14db62740a1e3 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Tue, 14 Jan 2025 16:21:37 -0600
Subject: [PATCH 140/161] BL-924

---
 .../runtime/bifs/global/system/IncludeTestAgain.cfm  |  1 +
 .../runtime/bifs/global/system/IncludeTestAgain2.cfm |  1 +
 .../runtime/components/system/IncludeTest.java       | 12 ++++++++++++
 3 files changed, 14 insertions(+)
 create mode 100644 src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain.cfm
 create mode 100644 src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain2.cfm

diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain.cfm b/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain.cfm
new file mode 100644
index 000000000..7e94054c5
--- /dev/null
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain.cfm
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain2.cfm b/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain2.cfm
new file mode 100644
index 000000000..29efbfd17
--- /dev/null
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/IncludeTestAgain2.cfm
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/IncludeTest.java b/src/test/java/ortus/boxlang/runtime/components/system/IncludeTest.java
index b52f9b0c2..fc4303fa5 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/IncludeTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/IncludeTest.java
@@ -117,6 +117,18 @@ public void testCanIncludeTemplateTagDotDotSlash() {
 		assertThat( variables.get( result ).toString().contains( "IncludeTest.cfs" ) ).isTrue();
 	}
 
+	@DisplayName( "It can include template tag Dot Dot Slash again" )
+	@Test
+	public void testCanIncludeTemplateTagDotDotSlashAgain() {
+
+		instance.executeSource(
+		    """
+		    
+		     """,
+		    context, BoxSourceType.CFTEMPLATE );
+		assertThat( variables.getAsString( result ) ).contains( "IncludeTestAgain2.cfm" );
+	}
+
 	@DisplayName( "It can include template BL tag" )
 	@Test
 	public void testCanIncludeTemplateBLTag() {

From 8d5c27be58ae67176c0b595749d29c1346a5eadf Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 23:35:53 +0100
Subject: [PATCH 141/161] BL-938 #resolve

Update the getOrCreateSession() to verify if the session has expired, and if so, rotate it.
---
 .../runtime/application/Application.java      | 10 +++--
 .../boxlang/runtime/application/Session.java  | 39 ++++++++++++++++++-
 2 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/application/Application.java b/src/main/java/ortus/boxlang/runtime/application/Application.java
index 9603bbfa0..e32cacffd 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Application.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Application.java
@@ -389,23 +389,25 @@ public Session getOrCreateSession( Key ID, RequestBoxContext context ) {
 		} else {
 			timeoutDuration = Duration.ofSeconds( LongCaster.cast( sessionTimeout ) );
 		}
+		// Dumb Java! It needs a final variable to use in the lambda
+		final Duration	finalTimeoutDuration	= timeoutDuration;
 
 		// logger.debug( "**** getOrCreateSession {} Timeout {} ", ID, timeoutDuration );
 
 		// Get or create the session
-		Session targetSession = ( Session ) this.sessionsCache.getOrSet(
+		Session			targetSession			= ( Session ) this.sessionsCache.getOrSet(
 		    cacheKey,
-		    () -> new Session( ID, this ),
+		    () -> new Session( ID, this, finalTimeoutDuration ),
 		    timeoutDuration,
 		    timeoutDuration
 		);
 
 		// Is the session still valid?
-		if ( targetSession.isShutdown() ) {
+		if ( targetSession.isShutdown() || targetSession.isExpired() ) {
 			// If not, remove it
 			this.sessionsCache.clear( cacheKey );
 			// And create a new one
-			targetSession = new Session( ID, this );
+			targetSession = new Session( ID, this, finalTimeoutDuration );
 			this.sessionsCache.set( cacheKey, targetSession, timeoutDuration, timeoutDuration );
 		}
 
diff --git a/src/main/java/ortus/boxlang/runtime/application/Session.java b/src/main/java/ortus/boxlang/runtime/application/Session.java
index 116a3a44a..927d0f5b6 100644
--- a/src/main/java/ortus/boxlang/runtime/application/Session.java
+++ b/src/main/java/ortus/boxlang/runtime/application/Session.java
@@ -18,6 +18,7 @@
 package ortus.boxlang.runtime.application;
 
 import java.io.Serializable;
+import java.time.Duration;
 
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.context.IBoxContext;
@@ -73,6 +74,11 @@ public class Session implements Serializable {
 	 */
 	private Key					applicationName	= null;
 
+	/**
+	 * The timeout for this session
+	 */
+	private Duration			timeout;
+
 	/**
 	 * --------------------------------------------------------------------------
 	 * Constructor(s)
@@ -84,9 +90,11 @@ public class Session implements Serializable {
 	 *
 	 * @param ID          The ID of this session
 	 * @param application The application that this session belongs to
+	 * @param timeout     The timeout for this session when created
 	 */
-	public Session( Key ID, Application application ) {
+	public Session( Key ID, Application application, Duration timeout ) {
 		this.ID					= ID;
+		this.timeout			= timeout;
 		this.applicationName	= application.getName();
 		this.sessionScope		= new SessionScope();
 		DateTime timeNow = new DateTime();
@@ -205,6 +213,13 @@ public String getCacheKey() {
 		return buildCacheKey( this.ID, this.applicationName );
 	}
 
+	/**
+	 * Get the registered timeout for this session
+	 */
+	public Duration getTimeout() {
+		return this.timeout;
+	}
+
 	/**
 	 * Shutdown the session
 	 *
@@ -288,4 +303,26 @@ public IStruct asStruct() {
 		);
 	}
 
+	/**
+	 * Verifies if the session has expired or not
+	 */
+	/**
+	 * Has this application expired.
+	 * We look at the application start time and the application timeout to determine if it has expired
+	 *
+	 * @return True if the application has expired, false otherwise
+	 */
+	public boolean isExpired() {
+		// If the session scope doesn't have a last visit, then it's expired
+		if ( this.sessionScope == null || !this.sessionScope.containsKey( Key.lastVisit ) ) {
+			return true;
+		}
+
+		// If the start time + the duration is before now, then it's expired
+		DateTime lastVisit = ( DateTime ) this.sessionScope.get( Key.lastVisit );
+		// Example: 10:00 + 1 hour = 11:00, now is 11:01, so it's expired : true
+		// Example: 10:00 + 1 hour = 11:00, now is 10:59, so it's not expired : false
+		return lastVisit.plus( this.timeout ).isBefore( new DateTime() );
+	}
+
 }

From f247347e11832c84ea78bddfb96808615812f7c2 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Tue, 14 Jan 2025 23:55:22 +0100
Subject: [PATCH 142/161] BL-621 #resolve

---
 .../boxlang/runtime/dynamic/casters/StructCaster.java      | 7 -------
 .../boxlang/runtime/dynamic/casters/StructCasterLoose.java | 6 ++++++
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java
index 44bab681e..3ddde6e60 100644
--- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java
+++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCaster.java
@@ -22,8 +22,6 @@
 import ortus.boxlang.runtime.interop.DynamicObject;
 import ortus.boxlang.runtime.scopes.ArgumentsScope;
 import ortus.boxlang.runtime.types.IStruct;
-import ortus.boxlang.runtime.types.Query;
-import ortus.boxlang.runtime.types.Struct;
 import ortus.boxlang.runtime.types.StructMapWrapper;
 import ortus.boxlang.runtime.types.exceptions.BoxCastException;
 import ortus.boxlang.runtime.types.exceptions.ExceptionUtil;
@@ -92,11 +90,6 @@ public static IStruct cast( Object object, Boolean fail ) {
 			return StructMapWrapper.of( ( Map ) map );
 		}
 
-		// Special Productivity Hack: If it's a Query object, take the first row and return it as a struct
-		if ( object instanceof Query query ) {
-			return query.isEmpty() ? new Struct() : query.getRowAsStruct( 0 );
-		}
-
 		if ( fail ) {
 			throw new BoxCastException(
 			    String.format( "Can't cast [%s] to a Struct.", object.getClass().getName() )
diff --git a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java
index 7a1790e78..c8c641593 100644
--- a/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java
+++ b/src/main/java/ortus/boxlang/runtime/dynamic/casters/StructCasterLoose.java
@@ -24,6 +24,7 @@
 import ortus.boxlang.runtime.bifs.global.decision.IsObject;
 import ortus.boxlang.runtime.interop.DynamicObject;
 import ortus.boxlang.runtime.types.IStruct;
+import ortus.boxlang.runtime.types.Query;
 import ortus.boxlang.runtime.types.Struct;
 import ortus.boxlang.runtime.types.exceptions.BoxCastException;
 
@@ -84,6 +85,11 @@ public static IStruct cast( Object object, Boolean fail ) {
 			}
 		}
 
+		// Special Productivity Hack: If it's a Query object, take the first row and return it as a struct
+		if ( object instanceof Query query ) {
+			return query.isEmpty() ? new Struct() : query.getRowAsStruct( 0 );
+		}
+
 		// If it's a random Java class, then turn it into a struct!!
 		if ( IsObject.isObject( object ) ) {
 			IStruct			thisResult	= new Struct();

From 60055d71f283d64603ef432b1be491ec9a961546 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Tue, 14 Jan 2025 18:13:48 -0600
Subject: [PATCH 143/161] BL-939

---
 src/main/antlr/BoxScriptGrammar.g4            |  20 ++-
 src/main/antlr/BoxScriptLexer.g4              | 141 +++++++++---------
 .../ast/visitor/PrettyPrintBoxVisitor.java    |   2 +-
 .../compiler/parser/BoxParserControl.java     | 108 +++++++-------
 .../compiler/toolchain/BoxVisitor.java        |  15 +-
 .../TestCases/components/BoxTemplateTest.java |  74 ++++-----
 .../TestCases/phase1/LabeledLoopTest.java     |  14 +-
 .../TestCases/phase1/files/Application.bx     |   8 +-
 .../TestCases/phase3/ApplicationTest.java     |  22 +--
 src/test/java/TestCases/phase3/ClassTest.java |   4 +-
 src/test/java/external/TestBoxTest.java       |   2 +-
 .../boxlang/compiler/FileTesterTest.java      |  18 +--
 .../global/jdbc/PreserveSingleQuotesTest.java |  20 +--
 .../bifs/global/math/RandRangeTest.java       |  52 +++----
 .../global/system/ApplicationRestartTest.java |   2 +-
 .../system/ApplicationStartTimeTest.java      |   4 +-
 .../global/system/ApplicationStopTest.java    |   4 +-
 .../global/system/RunThreadInContextTest.java |  22 +--
 .../global/system/SessionInvalidateTest.java  |  12 +-
 .../bifs/global/system/SessionRotateTest.java |   2 +-
 .../global/system/SessionStartTimeTest.java   |   6 +-
 .../runtime/components/cache/CacheTest.java   |  26 ++--
 .../runtime/components/debug/TimerTest.java   |  72 ++++-----
 .../runtime/components/io/DirectoryTest.java  |  32 ++--
 .../runtime/components/io/FileTest.java       |  16 +-
 .../runtime/components/jdbc/QueryTest.java    |   4 +-
 .../runtime/components/net/HTTPTest.java      |  46 +++---
 .../runtime/components/system/DumpTest.java   |   2 +-
 .../runtime/components/system/ExitTest.java   |  12 +-
 .../runtime/components/system/InvokeTest.java |  22 +--
 .../runtime/components/system/LogTest.java    |   2 +-
 .../runtime/components/system/LoopTest.java   |  66 ++++----
 .../runtime/components/system/ModuleTest.java |  17 ++-
 .../components/system/SaveContentTest.java    |   2 +-
 34 files changed, 449 insertions(+), 422 deletions(-)

diff --git a/src/main/antlr/BoxScriptGrammar.g4 b/src/main/antlr/BoxScriptGrammar.g4
index cd2d45f6e..b421e5335 100644
--- a/src/main/antlr/BoxScriptGrammar.g4
+++ b/src/main/antlr/BoxScriptGrammar.g4
@@ -22,8 +22,11 @@ identifier: IDENTIFIER | reservedKeyword
 componentName
     :
     // Ask the component service if the component exists and verify that this context is actually a component.
-    { isComponent(_input) }? identifier
-    //identifier
+    // { isComponent(_input) }? identifier
+    identifier
+    ;
+
+specialComponentName: TRANSACTION | LOCK | THREAD | ABORT | EXIT | PARAM
     ;
 
 // These are reserved words in the lexer, but are allowed to be an indentifer (variable name, method name)
@@ -92,6 +95,11 @@ reservedKeyword
     | VAR
     | WHEN
     | WHILE
+    | TRANSACTION
+    | LOCK
+    | THREAD
+    | ABORT
+    | EXIT
     ;
 
 reservedOperators
@@ -313,9 +321,11 @@ not: NOT expression
 
 // bx:http url="google.com" {}?
 component
-    :
-    // COMPONENT_PREFIX componentName componentAttribute* (normalStatementBlock | SEMICOLON)
-    componentName componentAttribute* (normalStatementBlock | SEMICOLON)
+    : (( COMPONENT_PREFIX componentName) | specialComponentName) componentAttribute* (
+        normalStatementBlock
+        | SEMICOLON
+    )
+    //componentName componentAttribute* (normalStatementBlock | SEMICOLON)
     ;
 
 componentAttribute: identifier ((EQUALSIGN | COLON) expression)?
diff --git a/src/main/antlr/BoxScriptLexer.g4 b/src/main/antlr/BoxScriptLexer.g4
index 08268248f..5f2d1b1b5 100644
--- a/src/main/antlr/BoxScriptLexer.g4
+++ b/src/main/antlr/BoxScriptLexer.g4
@@ -19,74 +19,79 @@ options {
     caseInsensitive = true;
 }
 
-ABSTRACT   : 'ABSTRACT';
-ANY        : 'ANY';
-ARRAY      : 'ARRAY';
-AS         : 'AS';
-ASSERT     : 'ASSERT';
-BOOLEAN    : 'BOOLEAN';
-BREAK      : 'BREAK';
-CASE       : 'CASE';
-CASTAS     : 'CASTAS';
-CATCH      : 'CATCH';
-CONTAIN    : 'CONTAIN';
-CONTAINS   : 'CONTAINS';
-CONTINUE   : 'CONTINUE';
-DEFAULT    : 'DEFAULT';
-DO         : 'DO';
-DOES       : 'DOES';
-ELIF       : 'ELIF';
-ELSE       : 'ELSE';
-EQV        : 'EQV';
-FALSE      : 'FALSE';
-FINAL      : 'FINAL';
-FINALLY    : 'FINALLY';
-FOR        : 'FOR';
-FUNCTION   : 'FUNCTION';
-GREATER    : 'GREATER';
-IF         : 'IF';
-IMP        : 'IMP';
-IMPORT     : 'IMPORT';
-IN         : 'IN';
-INCLUDE    : 'INCLUDE';
-INSTANCEOF : 'INSTANCEOF';
-INTERFACE  : 'INTERFACE';
-IS         : 'IS';
-JAVA       : 'JAVA';
-LESS       : 'LESS';
-MESSAGE    : 'MESSAGE';
-MOD        : 'MOD';
-NEW        : 'NEW';
-NULL       : 'NULL';
-NUMERIC    : 'NUMERIC';
-PARAM      : 'PARAM';
-PACKAGE    : 'PACKAGE';
-PRIVATE    : 'PRIVATE';
-PROPERTY   : 'PROPERTY';
-PUBLIC     : 'PUBLIC';
-QUERY      : 'QUERY';
-REMOTE     : 'REMOTE';
-REQUIRED   : 'REQUIRED';
-RETHROW    : 'RETHROW';
-RETURN     : 'RETURN';
-REQUEST    : 'REQUEST';
-SERVER     : 'SERVER';
-SETTING    : 'SETTING';
-STATIC     : 'STATIC';
-STRING     : 'STRING';
-STRUCT     : 'STRUCT';
-SWITCH     : 'SWITCH';
-THAN       : 'THAN';
-THROW      : 'THROW';
-TO         : 'TO';
-TRUE       : 'TRUE';
-TRY        : 'TRY';
-TYPE       : 'TYPE';
-VAR        : 'VAR';
-VARIABLES  : 'VARIABLES';
-WHEN       : 'WHEN';
-WHILE      : 'WHILE';
-XOR        : 'XOR';
+ABORT       : 'ABORT';
+ABSTRACT    : 'ABSTRACT';
+ANY         : 'ANY';
+ARRAY       : 'ARRAY';
+AS          : 'AS';
+ASSERT      : 'ASSERT';
+BOOLEAN     : 'BOOLEAN';
+BREAK       : 'BREAK';
+CASE        : 'CASE';
+CASTAS      : 'CASTAS';
+CATCH       : 'CATCH';
+CONTAIN     : 'CONTAIN';
+CONTAINS    : 'CONTAINS';
+CONTINUE    : 'CONTINUE';
+DEFAULT     : 'DEFAULT';
+DO          : 'DO';
+DOES        : 'DOES';
+ELIF        : 'ELIF';
+ELSE        : 'ELSE';
+EQV         : 'EQV';
+EXIT        : 'EXIT';
+FALSE       : 'FALSE';
+FINAL       : 'FINAL';
+FINALLY     : 'FINALLY';
+FOR         : 'FOR';
+FUNCTION    : 'FUNCTION';
+GREATER     : 'GREATER';
+IF          : 'IF';
+IMP         : 'IMP';
+IMPORT      : 'IMPORT';
+IN          : 'IN';
+INCLUDE     : 'INCLUDE';
+INSTANCEOF  : 'INSTANCEOF';
+INTERFACE   : 'INTERFACE';
+IS          : 'IS';
+JAVA        : 'JAVA';
+LESS        : 'LESS';
+LOCK        : 'LOCK';
+MESSAGE     : 'MESSAGE';
+MOD         : 'MOD';
+NEW         : 'NEW';
+NULL        : 'NULL';
+NUMERIC     : 'NUMERIC';
+PARAM       : 'PARAM';
+PACKAGE     : 'PACKAGE';
+PRIVATE     : 'PRIVATE';
+PROPERTY    : 'PROPERTY';
+PUBLIC      : 'PUBLIC';
+QUERY       : 'QUERY';
+REMOTE      : 'REMOTE';
+REQUIRED    : 'REQUIRED';
+RETHROW     : 'RETHROW';
+RETURN      : 'RETURN';
+REQUEST     : 'REQUEST';
+SERVER      : 'SERVER';
+SETTING     : 'SETTING';
+STATIC      : 'STATIC';
+STRING      : 'STRING';
+STRUCT      : 'STRUCT';
+SWITCH      : 'SWITCH';
+THAN        : 'THAN';
+THREAD      : 'THREAD';
+THROW       : 'THROW';
+TO          : 'TO';
+TRANSACTION : 'TRANSACTION';
+TRUE        : 'TRUE';
+TRY         : 'TRY';
+TYPE        : 'TYPE';
+VAR         : 'VAR';
+VARIABLES   : 'VARIABLES';
+WHEN        : 'WHEN';
+WHILE       : 'WHILE';
+XOR         : 'XOR';
 
 CLASS: 'CLASS';
 
diff --git a/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java b/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java
index 8cd29063c..e1777d843 100644
--- a/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java
+++ b/src/main/java/ortus/boxlang/compiler/ast/visitor/PrettyPrintBoxVisitor.java
@@ -1680,7 +1680,7 @@ public void visit( BoxComponent node ) {
 				print( ">" );
 			}
 		} else {
-			// print( "bx:" );
+			print( "bx:" );
 			print( node.getName() );
 			for ( var attr : node.getAttributes() ) {
 				print( " " );
diff --git a/src/main/java/ortus/boxlang/compiler/parser/BoxParserControl.java b/src/main/java/ortus/boxlang/compiler/parser/BoxParserControl.java
index 404c6ffe6..aad2b60b5 100644
--- a/src/main/java/ortus/boxlang/compiler/parser/BoxParserControl.java
+++ b/src/main/java/ortus/boxlang/compiler/parser/BoxParserControl.java
@@ -18,12 +18,10 @@
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.DEFAULT;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.DO;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.DOES;
-import static ortus.boxlang.parser.antlr.BoxScriptGrammar.DOT;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.ELIF;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.ELSE;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.EQ;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.EQUAL;
-import static ortus.boxlang.parser.antlr.BoxScriptGrammar.EQUALSIGN;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.EQV;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.FALSE;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.FINAL;
@@ -44,7 +42,6 @@
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.INTERFACE;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.IS;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.JAVA;
-import static ortus.boxlang.parser.antlr.BoxScriptGrammar.LBRACKET;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.LE;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.LESS;
 import static ortus.boxlang.parser.antlr.BoxScriptGrammar.LPAREN;
@@ -132,57 +129,60 @@ private boolean isType( int type ) {
 	 * @return true if the stream represents a component
 	 */
 	protected boolean isComponent( TokenStream input ) {
-
-		var nextToken = input.LT( 1 );
-
-		// Short circuit if not an identifier
-		if ( !identifiers.contains( nextToken.getType() ) ) {
-			return false;
-		}
-
-		var tokText = input.LT( 1 ).getText();
-
-		// It is not a component if it is not registered in the component service
-		if ( !componentService.hasComponent( tokText ) ) {
-			return false;
-		}
-
-		// If a function call, then ( will be next so reject the component
-		if ( input.LT( 2 ).getType() == LPAREN )
-			return false;
-
-		// If array access, then [ will be next so reject the component
-		if ( input.LT( 2 ).getType() == LBRACKET )
-			return false;
-
-		// Some components accept a type parameter, such as PARAM and if so we let them got through
-		// the standard rules and not component
-		//
-		// PARAM String ....
-		//
-		// But we also see components where teh first annotation is a keyword used as an id, so
-		// we have to assume that they are components
-		//
-		// SomeComp CLASS="dfdsffds"
-		//
-		// NB: It is possible that we could just check LT(1) == "PARAM" - but it is not clear to
-		// me that PARAM always should be parsed using its own rule. If so, you can simplify this
-		// method by just checking for PARAM.
-
-		if ( isType( input.LT( 2 ).getType() ) ) {
-			// If what looks like a type is actually assigned to, then it is in fact a component
-			return input.LT( 3 ).getType() == EQUALSIGN;
-		}
-
-		// Sill looks like a component but components can't be named x.access, so it is a FQN of some sort if that is the name
-		if ( input.LT( 2 ).getType() == DOT ) {
-			return false;
-		}
-		// param x.y - component attributes cannot be FQN, so this is param or something similar
-		return input.LT( 3 ).getType() != DOT;
-
-		// Having elimnated all possible ways that this is not a component,
-		// we know it must be a component
+		// The bx: prefix should remove this ambiguity
+		return true;
+		/*
+		 * var nextToken = input.LT( 1 );
+		 * 
+		 * // Short circuit if not an identifier
+		 * if ( !identifiers.contains( nextToken.getType() ) ) {
+		 * return false;
+		 * }
+		 * 
+		 * var tokText = input.LT( 1 ).getText();
+		 * 
+		 * // It is not a component if it is not registered in the component service
+		 * if ( !componentService.hasComponent( tokText ) ) {
+		 * return false;
+		 * }
+		 * 
+		 * // If a function call, then ( will be next so reject the component
+		 * if ( input.LT( 2 ).getType() == LPAREN )
+		 * return false;
+		 * 
+		 * // If array access, then [ will be next so reject the component
+		 * if ( input.LT( 2 ).getType() == LBRACKET )
+		 * return false;
+		 * 
+		 * // Some components accept a type parameter, such as PARAM and if so we let them got through
+		 * // the standard rules and not component
+		 * //
+		 * // PARAM String ....
+		 * //
+		 * // But we also see components where teh first annotation is a keyword used as an id, so
+		 * // we have to assume that they are components
+		 * //
+		 * // SomeComp CLASS="dfdsffds"
+		 * //
+		 * // NB: It is possible that we could just check LT(1) == "PARAM" - but it is not clear to
+		 * // me that PARAM always should be parsed using its own rule. If so, you can simplify this
+		 * // method by just checking for PARAM.
+		 * 
+		 * if ( isType( input.LT( 2 ).getType() ) ) {
+		 * // If what looks like a type is actually assigned to, then it is in fact a component
+		 * return input.LT( 3 ).getType() == EQUALSIGN;
+		 * }
+		 * 
+		 * // Sill looks like a component but components can't be named x.access, so it is a FQN of some sort if that is the name
+		 * if ( input.LT( 2 ).getType() == DOT ) {
+		 * return false;
+		 * }
+		 * // param x.y - component attributes cannot be FQN, so this is param or something similar
+		 * return input.LT( 3 ).getType() != DOT;
+		 * 
+		 * // Having elimnated all possible ways that this is not a component,
+		 * // we know it must be a component
+		 */
 	}
 
 	/**
diff --git a/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java b/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java
index ecf3522c0..5e9901a8c 100644
--- a/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java
+++ b/src/main/java/ortus/boxlang/compiler/toolchain/BoxVisitor.java
@@ -447,11 +447,18 @@ public BoxStatement visitWhile( WhileContext ctx ) {
 
 	@Override
 	public BoxNode visitComponent( ComponentContext ctx ) {
-		var					pos			= tools.getPosition( ctx );
-		var					src			= tools.getSourceText( ctx );
+		var		pos	= tools.getPosition( ctx );
+		var		src	= tools.getSourceText( ctx );
 
-		String				name		= ctx.componentName().getText();
-		List	attributes	= Optional.ofNullable( ctx.componentAttribute() )
+		String	name;
+		// Any identifer prefixed with bx:
+		if ( ctx.componentName() != null ) {
+			name = ctx.componentName().getText();
+		} else {
+			// specical component name's like transaction which are allowed to not have bx: in front
+			name = ctx.specialComponentName().getText();
+		}
+		List attributes = Optional.ofNullable( ctx.componentAttribute() )
 		    .map( attributeList -> attributeList.stream().map( attribute -> ( BoxAnnotation ) attribute.accept( this ) ).collect( Collectors.toList() ) )
 		    .orElse( Collections.emptyList() );
 
diff --git a/src/test/java/TestCases/components/BoxTemplateTest.java b/src/test/java/TestCases/components/BoxTemplateTest.java
index 29bffbb94..3c57227ca 100644
--- a/src/test/java/TestCases/components/BoxTemplateTest.java
+++ b/src/test/java/TestCases/components/BoxTemplateTest.java
@@ -230,18 +230,18 @@ public void testComponentIsland() {
 	public void testComponentIslandBufferOrder() {
 		instance.executeSource(
 		    """
-		      setting enableOutputOnly=true;
-		        	echo( "I am a script" )
-
-		        	```
-		        	
-		        	hello
-		        	```
-
-		        	// Now I am back in scripts
-		        	echo( "scripts again" )
-		    result = getBoxContext().getBuffer().toString()
-		        """, context, BoxSourceType.BOXSCRIPT );
+		    bx:setting enableOutputOnly=true;
+		          	echo( "I am a script" )
+
+		          	```
+		          	
+		          	hello
+		          	```
+
+		          	// Now I am back in scripts
+		          	echo( "scripts again" )
+		      result = getBoxContext().getBuffer().toString()
+		          """, context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.get( result ) ).isEqualTo( "I am a scripthelloscripts again" );
 	}
 
@@ -789,26 +789,26 @@ public void testGenericComponentsDanglingEnd() {
 	public void testGenericComponentsInScript() {
 		instance.executeSource(
 		    """
-		    http url="http://google.com" throwOnTimeout=true {
-		    	foo = "bar";
-		    	baz=true;
-		    }
+		       bx:http url="http://google.com" throwOnTimeout=true {
+		       	foo = "bar";
+		       	baz=true;
+		       }
 
-		    http url="http://google.com" throwOnTimeout=true;
+		    bx:http url="http://google.com" throwOnTimeout=true;
 
-		                  """,
+		                     """,
 		    context, BoxSourceType.BOXSCRIPT );
 	}
 
 	@Test
-	public void testNonExistentcComponentsInScript() {
+	public void testNonExistentComponentsInScript() {
 		Throwable e = assertThrows( BoxRuntimeException.class, () -> instance.executeSource(
 		    """
-		    brad {
-		    }
-		          """,
+		    bx:brad {
+		      }
+		            """,
 		    context, BoxSourceType.BOXSCRIPT ) );
-		assertThat( e.getMessage() ).contains( "[brad] was not located" );
+		assertThat( e.getMessage() ).contains( "[brad] could not be found" );
 	}
 
 	@Test
@@ -896,13 +896,13 @@ public void testLoopConditionExpr() {
 	public void testLoopConditionScript() {
 		instance.executeSource(
 		    """
-		      result = "";
-		    counter=0;
-		    		 loop condition="counter LT 5" {
-		    			 counter++
-		    			result &= counter
-		    	}
-		    	""", context, BoxSourceType.BOXSCRIPT );
+		         result = "";
+		       counter=0;
+		    bx:loop condition="counter LT 5" {
+		       			 counter++
+		       			result &= counter
+		       	}
+		       	""", context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.get( result ) ).isEqualTo( "12345" );
 	}
 
@@ -910,13 +910,13 @@ public void testLoopConditionScript() {
 	public void testLoopConditionExprScript() {
 		instance.executeSource(
 		    """
-		      result = "";
-		    counter=0;
-		    		 loop condition="#counter LT 5#" {
-		    			 counter++
-		    			result &= counter
-		    	}
-		    	""", context, BoxSourceType.BOXSCRIPT );
+		         result = "";
+		       counter=0;
+		    bx:loop condition="#counter LT 5#" {
+		       			 counter++
+		       			result &= counter
+		       	}
+		       	""", context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.get( result ) ).isEqualTo( "12345" );
 	}
 
diff --git a/src/test/java/TestCases/phase1/LabeledLoopTest.java b/src/test/java/TestCases/phase1/LabeledLoopTest.java
index a67f55369..5f221a110 100644
--- a/src/test/java/TestCases/phase1/LabeledLoopTest.java
+++ b/src/test/java/TestCases/phase1/LabeledLoopTest.java
@@ -180,13 +180,13 @@ public void testSimpleLabeledLoop() {
 
 		instance.executeSource(
 		    """
-		    result = 0
-		    	loop condition="true" label="mylabel" {
-		    		result ++
-		    		break mylabel;
-		    		result ++
-		    	}
-		         """,
+		       result = 0
+		    bx:loop condition="true" label="mylabel" {
+		       		result ++
+		       		break mylabel;
+		       		result ++
+		       	}
+		            """,
 		    context );
 		assertThat( variables.get( result ) ).isEqualTo( 1 );
 	}
diff --git a/src/test/java/TestCases/phase1/files/Application.bx b/src/test/java/TestCases/phase1/files/Application.bx
index 6e9c18a88..23772dcfc 100644
--- a/src/test/java/TestCases/phase1/files/Application.bx
+++ b/src/test/java/TestCases/phase1/files/Application.bx
@@ -1,19 +1,19 @@
 class output=true {
 	echo("application pseudoconstructor echo");
-	flush;
+	bx:flush;
 	
 	function onRequestStart() output=true {
 		echo("onRequestStart echo");
-		flush;
+		bx:flush;
 	}
 	
 	function onRequestEnd() output=true {
 		echo("onRequestEnd echo");
-		flush;
+		bx:flush;
 	}
 	
 	function onApplicationStart() output=true {
 		echo("onApplicationStart echo");
-		flush;
+		bx:flush;
 	}
 }
\ No newline at end of file
diff --git a/src/test/java/TestCases/phase3/ApplicationTest.java b/src/test/java/TestCases/phase3/ApplicationTest.java
index 2c26cddc8..057def21b 100644
--- a/src/test/java/TestCases/phase3/ApplicationTest.java
+++ b/src/test/java/TestCases/phase3/ApplicationTest.java
@@ -66,7 +66,7 @@ public void testBasicApplication() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application name="myAppsdfsdf" sessionmanagement="true";
+		        bx:application name="myAppsdfsdf" sessionmanagement="true";
 
 				result = application;
 				result2 = session;
@@ -93,7 +93,7 @@ public void testGetAppMeta() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application name="myAppsdfsdf2" sessionmanagement="true";
+		        bx:application name="myAppsdfsdf2" sessionmanagement="true";
 				result = GetApplicationMetadata();
 			""", context );
 		// @formatter:on
@@ -128,7 +128,7 @@ public void testJavaSettings() {
 		    """
 				import java.lang.Thread;
 
-		        application name="myJavaApp" javaSettings={
+		        bx:application name="myJavaApp" javaSettings={
 					loadPaths = [ "/src/test/resources/libs" ],
 					reloadOnChange = true
 				 };
@@ -175,7 +175,7 @@ public void testJavaSettingsPaths() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application name="myJavaApp" javaSettings={
+		        bx:application name="myJavaApp" javaSettings={
 					loadPaths = [ "/src/test/resources/libs/helloworld.jar" ],
 					reloadOnChange = true
 				 };
@@ -193,7 +193,7 @@ public void testJavaSettingsBadPaths() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application name="myJavaApp" javaSettings={
+		        bx:application name="myJavaApp" javaSettings={
 					loadPaths = [ "\\src\\test\\resources\\libs\\helloworld.jar" ],
 					reloadOnChange = true
 				 };
@@ -220,7 +220,7 @@ public void testJavaSettingsRelativePaths() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application name="myJavaApp" javaSettings={
+		        bx:application name="myJavaApp" javaSettings={
 					loadPaths = [ "libs/helloworld.jar" ],
 					reloadOnChange = true
 				 };
@@ -238,7 +238,7 @@ public void testJavaSettingsMappings() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application
+		        bx:application
 					name="myJavaAppWithMappings"
 					mappings = { "/javalib": "/src/test/resources/libs/" }
 					javaSettings={
@@ -259,7 +259,7 @@ public void testDatasourceDeclaration() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application
+		        bx:application
 					name="myAppWithDatasource"
 					datasources = { 
 						mysql = {
@@ -284,7 +284,7 @@ public void testTimezoneDeclaration() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application
+		        bx:application
 					name="myAppWithDatasource"
 					timezone="America/Los_Angeles";
 			""", context );
@@ -301,7 +301,7 @@ public void testUpdateApplicationWithoutName() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		        application
+		        bx:application
 					name="testUpdateApplicationWithoutName"
 					sessionmanagement="true";
 
@@ -311,7 +311,7 @@ public void testUpdateApplicationWithoutName() {
 					"/UpdateApplicationWithoutName" : "/src/test/resources/libs/"
 				}
 
-				application
+				bx:application
 					action        ="update"
 					mappings      ="#newMappings#";
 
diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java
index 9e877bb38..4831a73db 100644
--- a/src/test/java/TestCases/phase3/ClassTest.java
+++ b/src/test/java/TestCases/phase3/ClassTest.java
@@ -417,7 +417,7 @@ public void testBasicClassFileViaComponentPath() {
 		instance.executeSource(
 		    """
 				newClassPaths = getApplicationMetadata().classPaths.append( expandPath( "/src/test/java/TestCases/phase3" ) );
-				application classPaths=newClassPaths;
+				bx:application classPaths=newClassPaths;
 				cfc = new MyClass();
 				// execute public method
 				result = cfc.foo();
@@ -433,7 +433,7 @@ public void testBasicClassFileViaComponentPathSubDir() {
 		instance.executeSource(
 		    """
 				newClassPaths = getApplicationMetadata().classPaths.append( expandPath( "/src/test/java/TestCases" ) );
-				application classPaths=newClassPaths;
+				bx:application classPaths=newClassPaths;
 				import phase3.MyClass as bradClass;
 				cfc = new bradClass();
 				// execute public method
diff --git a/src/test/java/external/TestBoxTest.java b/src/test/java/external/TestBoxTest.java
index 2b67c6352..b27fe8089 100644
--- a/src/test/java/external/TestBoxTest.java
+++ b/src/test/java/external/TestBoxTest.java
@@ -73,7 +73,7 @@ Stream runDynamicTests() {
 				println( response );
 			}
 
-			application name="testbox runner" mappings={"/testbox":testboxDir};
+			bx:application name="testbox runner" mappings={"/testbox":testboxDir};
 
 			result = new testbox.system.TestBox().runRaw(  directory='src.test.java.external.specs' ).getMemento();
 
diff --git a/src/test/java/ortus/boxlang/compiler/FileTesterTest.java b/src/test/java/ortus/boxlang/compiler/FileTesterTest.java
index f84a27beb..4bd533c3c 100644
--- a/src/test/java/ortus/boxlang/compiler/FileTesterTest.java
+++ b/src/test/java/ortus/boxlang/compiler/FileTesterTest.java
@@ -105,15 +105,15 @@ public void testSaveContent() {
 		// instance.useJavaBoxpiler();
 		instance.executeSource(
 		    """
-		    y = []
-		       savecontent variable="x" {
-		       	if( true ){
-		       		loop array="#y#" index = "i" {
-
-		       		}
-		       	}
-		       }
-		                                     """,
+		       y = []
+		    bx:savecontent variable="x" {
+		          	if( true ){
+		    		bx:loop array="#y#" index = "i" {
+
+		          		}
+		          	}
+		          }
+		                                        """,
 		    context );
 	}
 
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/PreserveSingleQuotesTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/PreserveSingleQuotesTest.java
index ac6b8da22..ab9838b15 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/PreserveSingleQuotesTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/PreserveSingleQuotesTest.java
@@ -66,7 +66,7 @@ public void testNameSQLInterpolatedTags() {
 	public void testNormalScript() {
 		instance.executeSource(
 		    """
-		    query name="result" {
+		    bx:query name="result" {
 		    	echo( "SELECT *
 		    	FROM developers
 		    	WHERE name = 'Bob O''Reily' ")
@@ -84,7 +84,7 @@ public void testEntireSQLInterpolatedScript() {
 		    sql = "SELECT *
 		    	FROM developers
 		    	WHERE name = 'Bob O''Reily'";
-		       query name="result" {
+		       bx:query name="result" {
 		    	   echo( preserveSingleQuotes( sql ) )
 		       }
 		    		  """,
@@ -97,13 +97,13 @@ public void testEntireSQLInterpolatedScript() {
 	public void testNameSQLInterpolatedScript() {
 		instance.executeSource(
 		    """
-		      name = "Bob O'Reily";
-		    	query name="result" {
-		    	 echo( "SELECT *
-		    FROM developers
-		    WHERE name = '#name#'" )
-		    	}
-		    		""",
+		       name = "Bob O'Reily";
+		    bx:query name="result" {
+		     	 echo( "SELECT *
+		     FROM developers
+		     WHERE name = '#name#'" )
+		     	}
+		     		""",
 		    context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.getAsQuery( result ).size() ).isEqualTo( 1 );
 		assertThat( variables.getAsQuery( result ).getRowAsStruct( 0 ).get( "name" ) ).isEqualTo( "Bob O'Reily" );
@@ -114,7 +114,7 @@ public void testNameSQLConcatScript() {
 		instance.executeSource(
 		    """
 		    name = "Bob O'Reily";
-		    query name="result" {
+		    bx:query name="result" {
 		    	echo( "SELECT *
 		    	FROM developers
 		    	WHERE name = '" & name & "'" )
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/math/RandRangeTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/math/RandRangeTest.java
index b53b0cb78..49d3277b3 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/math/RandRangeTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/math/RandRangeTest.java
@@ -61,7 +61,7 @@ public void setupEach() {
 	public void testItReturnsARandomNumberInRange() {
 		instance.executeSource(
 		    """
-		    loop times=1000 {
+		    bx:loop times=1000 {
 		       	result = randRange( 0, 12 );
 		    	assert result >= 0;
 		    	assert result <= 12;
@@ -70,16 +70,16 @@ public void testItReturnsARandomNumberInRange() {
 
 		instance.executeSource(
 		    """
-		    loop times=1000 {
-		    	result = randRange( -12, 0 );
-		    	assert result >= -12;
-		    	assert result <= 0;
-		    }
-		    		 """, context );
+		    bx:loop times=1000 {
+		       	result = randRange( -12, 0 );
+		       	assert result >= -12;
+		       	assert result <= 0;
+		       }
+		       		 """, context );
 
 		instance.executeSource(
 		    """
-		    loop times=1000 {
+		    bx:loop times=1000 {
 		     result = randRange( 3.5, 4.9 );
 		     assert result >= 3;
 		     assert result <= 4;
@@ -89,13 +89,13 @@ public void testItReturnsARandomNumberInRange() {
 
 		instance.executeSource(
 		    """
-		    loop times=1000 {
-		     result = randRange( 100000000000000000000000, 100000000000000000001000 );
-		     assert result >= 100000000000000000000000;
-		     assert result <= 100000000000000000001000;
+		    bx:loop times=1000 {
+		        result = randRange( 100000000000000000000000, 100000000000000000001000 );
+		        assert result >= 100000000000000000000000;
+		        assert result <= 100000000000000000001000;
 
-		    }
-		       """, context );
+		       }
+		          """, context );
 	}
 
 	@DisplayName( "It returns a random number in range using an algorithm" )
@@ -103,12 +103,12 @@ public void testItReturnsARandomNumberInRange() {
 	public void testItReturnsARandomNumberInRangeWithAlgorithm() {
 		instance.executeSource(
 		    """
-		    loop times=1000 {
-		       	result = randRange( 0, 12, "SHA1PRNG" );
-		    	assert result >= 0;
-		    	assert result <= 12;
-		    }
-		       """, context );
+		    bx:loop times=1000 {
+		          	result = randRange( 0, 12, "SHA1PRNG" );
+		       	assert result >= 0;
+		       	assert result <= 12;
+		       }
+		          """, context );
 	}
 
 	@DisplayName( "It includes upper and lower bound" )
@@ -116,12 +116,12 @@ public void testItReturnsARandomNumberInRangeWithAlgorithm() {
 	public void testItIncludesUpperAndLowerBound() {
 		instance.executeSource(
 		    """
-		    result = []
-		       loop times=1000 {
-		          	result.append( randRange( 1, 3 ) );
-		    		 //result.append( rand() );
-		       }
-		          """, context );
+		      result = []
+		    bx:loop times=1000 {
+		            	result.append( randRange( 1, 3 ) );
+		      		 //result.append( rand() );
+		         }
+		            """, context );
 		assertThat( variables.getAsArray( result ) ).contains( 1L );
 		assertThat( variables.getAsArray( result ) ).contains( 2L );
 		assertThat( variables.getAsArray( result ) ).contains( 3L );
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationRestartTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationRestartTest.java
index c55b9591c..3c55db599 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationRestartTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationRestartTest.java
@@ -67,7 +67,7 @@ void setupEach() {
 	void testItCanStopAnApplication() {
 
 		instance.executeSource( """
-		                        application name="unit-test1" sessionmanagement="true";
+		                        bx:application name="unit-test1" sessionmanagement="true";
 		                           """, context );
 
 		Application targetApp = context.getParentOfType( ApplicationBoxContext.class ).getApplication();
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStartTimeTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStartTimeTest.java
index cb6359fbe..2a768e810 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStartTimeTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStartTimeTest.java
@@ -70,8 +70,8 @@ void testItCanStopAnApplication() {
 
 		instance.executeSource(
 		    """
-		    application name="unit-test2" sessionmanagement="true";
-		       """,
+		    bx:application name="unit-test2" sessionmanagement="true";
+		         """,
 		    context );
 
 		Application targetApp = context.getParentOfType( ApplicationBoxContext.class ).getApplication();
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStopTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStopTest.java
index 93bd87f13..4a32e8cf9 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStopTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/ApplicationStopTest.java
@@ -66,8 +66,8 @@ void testItCanStopAnApplication() {
 
 		runtime.executeSource(
 		    """
-		    application name="unit-test3" sessionmanagement="true";
-		       """,
+		    bx:application name="unit-test3" sessionmanagement="true";
+		         """,
 		    context );
 
 		Application targetApp = context.getParentOfType( ApplicationBoxContext.class ).getApplication();
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/RunThreadInContextTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/RunThreadInContextTest.java
index f1c5db280..426e23ab4 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/RunThreadInContextTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/RunThreadInContextTest.java
@@ -64,11 +64,11 @@ public void testCanRunCodeInContext() {
 	public void testCanRunCodeInApplication() {
 		instance.executeSource(
 		    """
-		    application name="myApp";
-		       runThreadInContext( applicationName="myApp", callback=()=>{
-		         	println( "running in application #application.applicationname#")
-		         });
-		            """,
+		    bx:application name="myApp";
+		         runThreadInContext( applicationName="myApp", callback=()=>{
+		           	println( "running in application #application.applicationname#")
+		           });
+		              """,
 		    context );
 	}
 
@@ -76,12 +76,12 @@ public void testCanRunCodeInApplication() {
 	public void testCanRunCodeInApplicationSession() {
 		instance.executeSource(
 		    """
-		       application name="myApp" sessionManagement=true;
-		    sessionID = session.jsessionID;
-		       runThreadInContext( applicationName="myApp", sessionID=sessionID, callback=()=>{
-		         	println( "running in application/session #application.applicationname#/#session.sessionID#")
-		         });
-		            """,
+		    bx:application name="myApp" sessionManagement=true;
+		      sessionID = session.jsessionID;
+		         runThreadInContext( applicationName="myApp", sessionID=sessionID, callback=()=>{
+		           	println( "running in application/session #application.applicationname#/#session.sessionID#")
+		           });
+		              """,
 		    context );
 	}
 }
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java
index cb1ccb50a..2408f56b3 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionInvalidateTest.java
@@ -64,12 +64,12 @@ public void setupEach() {
 	public void testBif() {
 		instance.executeSource(
 		    """
-		    application name="unit-test-sm" sessionmanagement="true";
-		         session.foo = "bar";
-		         initialSession = duplicate( session );
-		         SessionInvalidate();
-		         result = session;
-		         """,
+		    bx:application name="unit-test-sm" sessionmanagement="true";
+		           session.foo = "bar";
+		           initialSession = duplicate( session );
+		           SessionInvalidate();
+		           result = session;
+		           """,
 		    context );
 		IStruct initialSession = variables.getAsStruct( Key.of( "initialSession" ) );
 		assertFalse( variables.getAsStruct( result ).containsKey( Key.of( "foo" ) ) );
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionRotateTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionRotateTest.java
index 606077c41..3b677866f 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionRotateTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionRotateTest.java
@@ -64,7 +64,7 @@ public void testBif() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		     	application name="unit-test-sm" sessionmanagement="true";
+				bx:application name="unit-test-sm" sessionmanagement="true";
 
 				session.foo = "bar";
 				initialSession = duplicate( session );
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionStartTimeTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionStartTimeTest.java
index 3d2e458bd..68fbaa7ac 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionStartTimeTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/system/SessionStartTimeTest.java
@@ -67,9 +67,9 @@ public void testBif() {
 		}
 		instance.executeSource(
 		    """
-		    application name="unit-test-sm" sessionmanagement="true";
-		         result = sessionStartTime();
-		         """,
+		    bx:application name="unit-test-sm" sessionmanagement="true";
+		           result = sessionStartTime();
+		           """,
 		    context );
 
 		assertTrue( variables.get( result ) instanceof DateTime );
diff --git a/src/test/java/ortus/boxlang/runtime/components/cache/CacheTest.java b/src/test/java/ortus/boxlang/runtime/components/cache/CacheTest.java
index 1b19c5186..cfc256251 100644
--- a/src/test/java/ortus/boxlang/runtime/components/cache/CacheTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/cache/CacheTest.java
@@ -119,7 +119,7 @@ public void testComponentBX() {
 	public void testComponentScript() {
 		instance.executeSource(
 		    """
-		    cache action="cache" name="result" key="foo" value="bar";
+		    bx:cache action="cache" name="result" key="foo" value="bar";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -133,11 +133,11 @@ public void testComponentScript() {
 	public void testComponentPut() {
 		instance.executeSource(
 		    """
-		       cache action="put" key="foo" value="bar";
-		    cache action="get" key="foo" name="result";
-		       cache action="put" key="foo" value="baz";
-		    cache action="get" key="foo" name="result2";
-		       """,
+		    bx:cache action="put" key="foo" value="bar";
+		    bx:cache action="get" key="foo" name="result";
+		    bx:cache action="put" key="foo" value="baz";
+		    bx:cache action="get" key="foo" name="result2";
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof String );
@@ -151,10 +151,10 @@ public void testComponentPut() {
 	public void testComponentScriptComplex() {
 		instance.executeSource(
 		    """
-		    myStruct = { "foo" : "bar" };
-		       cache action="put" name="result" key="foo" value="#myStruct#";
-		    cache action="get" key="foo" name="result";
-		       """,
+		       myStruct = { "foo" : "bar" };
+		    bx:cache action="put" name="result" key="foo" value="#myStruct#";
+		    bx: cache action="get" key="foo" name="result";
+		          """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof IStruct );
@@ -256,9 +256,9 @@ public void testComponentBXDirectorySingleItemFlush() {
 		variables.put( Key.cacheName, testCacheKey.getName() );
 		instance.executeSource(
 		    """
-		       cache action="put" key="foo" cachename="#cachename#" value="bar";
-		    cache action="get" key="foo" cachename="#cachename#" name="result";
-		       cache action="flush" key="foo";
+		       bx:cache action="put" key="foo" cachename="#cachename#" value="bar";
+		    bx:cache action="get" key="foo" cachename="#cachename#" name="result";
+		       bx:cache action="flush" key="foo";
 		       """,
 		    context, BoxSourceType.BOXSCRIPT );
 
diff --git a/src/test/java/ortus/boxlang/runtime/components/debug/TimerTest.java b/src/test/java/ortus/boxlang/runtime/components/debug/TimerTest.java
index ee9dc9a5b..86eef2da3 100644
--- a/src/test/java/ortus/boxlang/runtime/components/debug/TimerTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/debug/TimerTest.java
@@ -99,10 +99,10 @@ public void testComponentBXM() {
 	public void testComponentBX() {
 		instance.executeSource(
 		    """
-		    timer type="comment" label="TimeIt"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer type="comment" label="TimeIt"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( baos.toString().length() > 0 );
@@ -115,10 +115,10 @@ public void testComponentBX() {
 	public void testComponentVariable() {
 		instance.executeSource(
 		    """
-		    timer variable="result"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer variable="result"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof Long );
@@ -130,10 +130,10 @@ public void testComponentVariable() {
 	public void testComponentVariableNS() {
 		instance.executeSource(
 		    """
-		    timer variable="result" unit="nano"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer variable="result" unit="nano"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof Long );
@@ -145,10 +145,10 @@ public void testComponentVariableNS() {
 	public void testComponentVariableMicroSec() {
 		instance.executeSource(
 		    """
-		    timer variable="result" unit="micro"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer variable="result" unit="micro"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof Long );
@@ -160,10 +160,10 @@ public void testComponentVariableMicroSec() {
 	public void testComponentVariableSec() {
 		instance.executeSource(
 		    """
-		    timer variable="result" unit="second"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer variable="result" unit="second"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof Long );
@@ -175,10 +175,10 @@ public void testComponentVariableSec() {
 	public void testComponentLabelOnly() {
 		instance.executeSource(
 		    """
-		    timer label="TimeIt"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer label="TimeIt"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( baos.toString() instanceof String );
@@ -192,10 +192,10 @@ public void testComponentLabelOnly() {
 	public void testComponentVariableDebug() {
 		instance.executeSource(
 		    """
-		    timer type="debug" label="TimeIt"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:timer type="debug" label="TimeIt"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( ExpressionInterpreter.getVariable( context, "request.debugInfo", true ) instanceof IStruct );
@@ -240,10 +240,10 @@ public void testComponentOutline() {
 	public void testStopwatchVariable() {
 		instance.executeSource(
 		    """
-		    stopwatch variable="result"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:stopwatch variable="result"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( variables.get( result ) instanceof Long );
@@ -255,10 +255,10 @@ public void testStopwatchVariable() {
 	public void testStopWatchLabelOnly() {
 		instance.executeSource(
 		    """
-		    stopwatch label="TimeIt"{
-		    	sleep(1);
-		    }
-		       """,
+		    bx:stopwatch label="TimeIt"{
+		      	sleep(1);
+		      }
+		         """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertTrue( baos.toString().length() > 0 );
diff --git a/src/test/java/ortus/boxlang/runtime/components/io/DirectoryTest.java b/src/test/java/ortus/boxlang/runtime/components/io/DirectoryTest.java
index 7177c1cea..1ec227160 100644
--- a/src/test/java/ortus/boxlang/runtime/components/io/DirectoryTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/io/DirectoryTest.java
@@ -94,7 +94,7 @@ public void testDirectoryCreateDefault() {
 		assertFalse( FileSystemUtil.exists( targetDirectory ) );
 		instance.executeSource(
 		    """
-		    directory action="Create" directory="#destination#";
+		    bx:directory action="Create" directory="#destination#";
 		    """,
 		    context );
 		assertTrue( FileSystemUtil.exists( targetDirectory ) );
@@ -125,8 +125,8 @@ public void testDirectoryCopyDefault() {
 		FileSystemUtil.write( testFile, "copy directory test!" );
 		instance.executeSource(
 		    """
-		    directory action="copy" directory="#source#" destination="#destination#";
-		    """,
+		    bx:directory action="copy" directory="#source#" destination="#destination#";
+		      """,
 		    context );
 		assertTrue( FileSystemUtil.exists( targetDirectory ) );
 		assertTrue( FileSystemUtil.exists( testFile ) );
@@ -143,8 +143,8 @@ public void testNoCreatePaths() {
 		    RuntimeException.class,
 		    () -> instance.executeSource(
 		        """
-		        directory action="Create" directory="#destination#" createPath=false;
-		             """,
+		        bx:directory action="Create" directory="#destination#" createPath=false;
+		                  """,
 		        context )
 		);
 	}
@@ -156,8 +156,8 @@ public void testIgnoreExistsFalse() {
 		assertFalse( FileSystemUtil.exists( targetDirectory ) );
 		instance.executeSource(
 		    """
-		    directory action="Create" directory="#destination#";
-		      """,
+		    bx:directory action="Create" directory="#destination#";
+		        """,
 		    context );
 		assertTrue( FileSystemUtil.exists( targetDirectory ) );
 		assertThrows(
@@ -216,9 +216,9 @@ public void testAllDirectoryListBif() {
 		assertTrue( FileSystemUtil.exists( testDirectory ) );
 		instance.executeSource(
 		    """
-		    println(variables.testDirectory)
-		       directory action="List" directory="#variables.testDirectory#" name="result" recurse=false listInfo="all";
-		       """,
+		       println(variables.testDirectory)
+		    bx:directory action="List" directory="#variables.testDirectory#" name="result" recurse=false listInfo="all";
+		          """,
 		    context );
 		var result = variables.get( Key.of( "result" ) );
 		assertTrue( result instanceof Query );
@@ -242,9 +242,9 @@ public void testAllDirectoryListBif() {
 		// Test for BL-202
 		instance.executeSource(
 		    """
-		    println(variables.testDirectory)
-		       directory directory="#variables.testDirectory#" name="result" recurse=false;
-		       """,
+		       println(variables.testDirectory)
+		    bx:directory directory="#variables.testDirectory#" name="result" recurse=false;
+		          """,
 		    context );
 		result = variables.get( Key.of( "result" ) );
 		assertTrue( result instanceof Query );
@@ -278,9 +278,9 @@ public void testNameDirectoryListBif() {
 		assertTrue( FileSystemUtil.exists( testDirectory ) );
 		instance.executeSource(
 		    """
-		    println(variables.testDirectory)
-		       directory action="List" directory="#variables.testDirectory#" name="result" recurse=true listInfo="name";
-		       """,
+		       println(variables.testDirectory)
+		    bx:directory action="List" directory="#variables.testDirectory#" name="result" recurse=true listInfo="name";
+		          """,
 		    context );
 		var result = variables.get( Key.of( "result" ) );
 		assertTrue( result instanceof Query );
diff --git a/src/test/java/ortus/boxlang/runtime/components/io/FileTest.java b/src/test/java/ortus/boxlang/runtime/components/io/FileTest.java
index a63eeafc9..7bf9f9b7c 100644
--- a/src/test/java/ortus/boxlang/runtime/components/io/FileTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/io/FileTest.java
@@ -111,7 +111,7 @@ public void testTextFileWriteScript() throws IOException {
 		variables.put( Key.of( "testFile" ), Path.of( testTextFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="write" file="#testFile#" output="I am writing!";
+		    bx:file action="write" file="#testFile#" output="I am writing!";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -159,7 +159,7 @@ public void testTextFileReadScript() throws IOException {
 		variables.put( Key.of( "testFile" ), Path.of( testTextFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="read" file="#testFile#" variable="readVariable"; variable="readVariable"
+		    bx:file action="read" file="#testFile#" variable="readVariable"; variable="readVariable"
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -177,7 +177,7 @@ public void testFileReadBinary() throws IOException {
 		variables.put( Key.of( "testFile" ), Path.of( testBinaryFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="readBinary" file="#testFile#" variable="readVariable";
+		    bx:file action="readBinary" file="#testFile#" variable="readVariable";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -195,7 +195,7 @@ public void testFileDelete() throws IOException {
 		variables.put( Key.of( "testFile" ), Path.of( testTextFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="delete" file="#testFile#";
+		    bx:file action="delete" file="#testFile#";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -249,7 +249,7 @@ public void testFileCopy() throws IOException {
 		variables.put( Key.of( "newFile" ), Path.of( copiedFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="copy" source="#testFile#" destination="#newFile#";
+		    bx:file action="copy" source="#testFile#" destination="#newFile#";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -312,7 +312,7 @@ public void testFileMove() throws IOException {
 		variables.put( Key.of( "newFile" ), Path.of( movedFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="move" source="#testFile#" destination="#newFile#";
+		    bx:file action="move" source="#testFile#" destination="#newFile#";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -375,7 +375,7 @@ public void testFileRename() throws IOException {
 		variables.put( Key.of( "newFile" ), Path.of( movedFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="rename" source="#testFile#" destination="#newFile#";
+		    bx:file action="rename" source="#testFile#" destination="#newFile#";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
@@ -393,7 +393,7 @@ public void testFileAppend() throws IOException {
 		variables.put( Key.of( "testFile" ), Path.of( testTextFile ).toAbsolutePath().toString() );
 		instance.executeSource(
 		    """
-		    file action="append" file="#testFile#" output="!";
+		    bx:file action="append" file="#testFile#" output="!";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 
diff --git a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java
index 4397a7eb8..8add69d37 100644
--- a/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/jdbc/QueryTest.java
@@ -280,7 +280,7 @@ public void testReturnTypeArray() {
 	public void testReturnTypeStruct() {
 		getInstance().executeSource(
 		    """
-		    query name="result" returntype="struct" columnKey="role" {
+		    bx:query name="result" returntype="struct" columnKey="role" {
 		    	echo( "SELECT * FROM developers ORDER BY id" );
 		    };
 		    """,
@@ -327,7 +327,7 @@ public void testReturnTypeStruct() {
 	public void testMissingColumnKey() {
 		BoxRuntimeException e = assertThrows( BoxRuntimeException.class, () -> getInstance().executeSource(
 		    """
-		    query name="result" returntype="struct" {
+		    bx:query name="result" returntype="struct" {
 		    	echo( "SELECT * FROM developers ORDER BY id" );
 		    };
 		    """,
diff --git a/src/test/java/ortus/boxlang/runtime/components/net/HTTPTest.java b/src/test/java/ortus/boxlang/runtime/components/net/HTTPTest.java
index 6c732fd6d..4fe6f8009 100644
--- a/src/test/java/ortus/boxlang/runtime/components/net/HTTPTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/net/HTTPTest.java
@@ -84,8 +84,8 @@ public void testCanMakeHTTPCallScript( WireMockRuntimeInfo wmRuntimeInfo ) {
 		String baseURL = wmRuntimeInfo.getHttpBaseUrl();
 
 		instance.executeSource( String.format( """
-		                                        http url="%s" {
-		                                            httpparam type="header" name="User-Agent" value="Mozilla";
+		                                        bx:http url="%s" {
+		                                            bx:httpparam type="header" name="User-Agent" value="Mozilla";
 		                                       }
 		                                       result = bxhttp;
 		                                        """, baseURL + "/posts/1" ),
@@ -104,7 +104,7 @@ public void testCanMakeHTTPCallScript( WireMockRuntimeInfo wmRuntimeInfo ) {
 	public void testCookiesInQuery( WireMockRuntimeInfo wmRuntimeInfo ) {
 		stubFor( get( "/cookies" ).willReturn( ok().withHeader( "Set-Cookie", "foo=bar;path=/;secure;samesite=none;httponly" ) ) );
 
-		instance.executeSource( String.format( "http url=\"%s\" {}", wmRuntimeInfo.getHttpBaseUrl() + "/cookies" ),
+		instance.executeSource( String.format( "bx:http url=\"%s\" {}", wmRuntimeInfo.getHttpBaseUrl() + "/cookies" ),
 		    context );
 
 		assertThat( variables.get( bxhttp ) ).isInstanceOf( IStruct.class );
@@ -127,7 +127,7 @@ public void testMultipleCookies( WireMockRuntimeInfo wmRuntimeInfo ) {
 		    .withHeader( "Set-Cookie", "one=two;max-age=2592000;domain=example.com" )
 		) );
 
-		instance.executeSource( String.format( "http url=\"%s\" {}", wmRuntimeInfo.getHttpBaseUrl() + "/cookies" ),
+		instance.executeSource( String.format( "bx:http url=\"%s\" {}", wmRuntimeInfo.getHttpBaseUrl() + "/cookies" ),
 		    context );
 
 		assertThat( variables.get( bxhttp ) ).isInstanceOf( IStruct.class );
@@ -152,9 +152,9 @@ public void testPostFormParams( WireMockRuntimeInfo wmRuntimeInfo ) {
 		        .willReturn( created().withBody( "{\"id\": 1, \"name\": \"foobar\", \"body\": \"lorem ipsum dolor\"}" ) ) );
 
 		instance.executeSource( String.format( """
-		                                       http method="POST" url="%s" {
-		                                       	httpparam type="formfield" name="name" value="foobar";
-		                                       	httpparam type="formfield" name="body" value="lorem ipsum dolor";
+		                                       bx:http method="POST" url="%s" {
+		                                       	bx:httpparam type="formfield" name="name" value="foobar";
+		                                       	bx:httpparam type="formfield" name="body" value="lorem ipsum dolor";
 		                                       }
 		                                       """, wmRuntimeInfo.getHttpBaseUrl() + "/posts" ), context );
 
@@ -177,9 +177,9 @@ public void testPostFormParamsMultipleValuesForOneName( WireMockRuntimeInfo wmRu
 
 		// @formatter:off
 		instance.executeSource( String.format( """
-			http method="POST" url="%s" {
-				httpparam type="formfield" name="tags" value="tag-a";
-				httpparam type="formfield" name="tags" value="tag-b";
+			bx:http method="POST" url="%s" {
+				bx:httpparam type="formfield" name="tags" value="tag-a";
+				bx:httpparam type="formfield" name="tags" value="tag-b";
 			}
 		""", wmRuntimeInfo.getHttpBaseUrl() + "/posts" ), context );
 		// @formatter:on
@@ -202,8 +202,8 @@ public void testPostJsonBody( WireMockRuntimeInfo wmRuntimeInfo ) {
 		        .willReturn( created().withBody( "{\"id\": 1, \"name\": \"foobar\", \"body\": \"lorem ipsum dolor\"}" ) ) );
 
 		instance.executeSource( String.format( """
-		                                       http method="POST" url="%s" {
-		                                       	httpparam type="body" value="#JSONSerialize( { 'name': 'foobar', 'body': 'lorem ipsum dolor' } )#";
+		                                       bx:http method="POST" url="%s" {
+		                                       	bx:httpparam type="body" value="#JSONSerialize( { 'name': 'foobar', 'body': 'lorem ipsum dolor' } )#";
 		                                       }
 		                                       """, wmRuntimeInfo.getHttpBaseUrl() + "/posts" ), context );
 
@@ -381,8 +381,8 @@ public void testGetWithParams( WireMockRuntimeInfo wmRuntimeInfo ) {
 
 		instance.executeSource(
 		    String.format( """
-		                    http url="%s" {
-		                   	 httpparam type="url" name="userId" value=1;
+		                    bx:http url="%s" {
+		                   	 bx:httpparam type="url" name="userId" value=1;
 		                   }
 		                   result = bxhttp;
 		                    """, wmRuntimeInfo.getHttpBaseUrl() + "/posts" ),
@@ -433,8 +433,8 @@ public void testGetWithParams( WireMockRuntimeInfo wmRuntimeInfo ) {
 	public void testBadGateway() {
 		// @formatter:off
 		instance.executeSource( """
-			http method="GET" url="https://does-not-exist.also-does-not-exist" {
-				httpparam type="header" name="User-Agent" value="HyperCFML/7.5.2";
+			bx:http method="GET" url="https://does-not-exist.also-does-not-exist" {
+				bx:httpparam type="header" name="User-Agent" value="HyperCFML/7.5.2";
 			}
 			result = bxhttp;
 		""", context );
@@ -469,8 +469,8 @@ public void testTimeout( WireMockRuntimeInfo wmRuntimeInfo ) {
 		String baseURL = wmRuntimeInfo.getHttpBaseUrl();
 		// @formatter:off
 		instance.executeSource( String.format( """
-			http timeout="1" method="GET" url="%s" {
-				httpparam type="header" name="User-Agent" value="HyperCFML/7.5.2";
+			bx:http timeout="1" method="GET" url="%s" {
+				bx:httpparam type="header" name="User-Agent" value="HyperCFML/7.5.2";
 			}
 			result = bxhttp;
 		""", baseURL + "/timeout" ), context );
@@ -508,8 +508,8 @@ public void testFiles( WireMockRuntimeInfo wmRuntimeInfo ) {
 		String baseURL = wmRuntimeInfo.getHttpBaseUrl();
 		// @formatter:off
 		instance.executeSource( String.format( """
-			http method="POST" url="%s" {
-				httpparam type="file" name="photo" file="/src/test/resources/chuck_norris.jpg";
+			bx:http method="POST" url="%s" {
+				bx:httpparam type="file" name="photo" file="/src/test/resources/chuck_norris.jpg";
 			}
 			result = bxhttp;
 		""", baseURL + "/files" ), context );
@@ -549,9 +549,9 @@ public void testMultipart( WireMockRuntimeInfo wmRuntimeInfo ) {
 		String baseURL = wmRuntimeInfo.getHttpBaseUrl();
 		// @formatter:off
 		instance.executeSource( String.format( """
-			http method="POST" url="%s" {
-				httpparam type="file" name="photo" file="/src/test/resources/chuck_norris.jpg";
-				httpparam type="formfield" name="joke" value="Chuck Norris can divide by zero.";
+			bx:http method="POST" url="%s" {
+				bx:httpparam type="file" name="photo" file="/src/test/resources/chuck_norris.jpg";
+				bx:httpparam type="formfield" name="joke" value="Chuck Norris can divide by zero.";
 			}
 			result = bxhttp;
 		""", baseURL + "/multipart" ), context );
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/DumpTest.java b/src/test/java/ortus/boxlang/runtime/components/system/DumpTest.java
index e16761d56..aa6387c74 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/DumpTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/DumpTest.java
@@ -113,7 +113,7 @@ public void testCanDumpScript() {
 		// @formatter:off
 		instance.executeSource(
 		    """
-		       	dump var="My Value" format="html";
+		       	bx:dump var="My Value" format="html";
 		    """,
 		    context );
 		// @formatter:on
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/ExitTest.java b/src/test/java/ortus/boxlang/runtime/components/system/ExitTest.java
index d1424a4dd..ee1fbbd53 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/ExitTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/ExitTest.java
@@ -173,7 +173,7 @@ public void testCanExitModuleStart() {
 		    result = "";
 		       request.exitWhen="start"
 		       request.exitMethod="exitTag"
-		          module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		        bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		         result &= "body";
 		       }
 		       result &= "aftermodule";
@@ -186,7 +186,7 @@ public void testCanExitModuleStart() {
 		    	 request.loopCount=0;
 		    result = "";
 		       request.exitMethod="exitTemplate"
-		          module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		          bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		         result &= "body";
 		       }
 		       result &= "aftermodule";
@@ -199,7 +199,7 @@ public void testCanExitModuleStart() {
 		    	 request.loopCount=0;
 		    result = "";
 		       request.exitMethod="loop"
-		          module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		          bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		         result &= "body";
 		       }
 		                     """,
@@ -215,7 +215,7 @@ public void testCanExitModuleEnd() {
 		    result = "";
 		       request.exitWhen="end"
 		       request.exitMethod="exitTag"
-		          module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		    bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		         result &= "body";
 		       }
 		       result &= "aftermodule";
@@ -228,7 +228,7 @@ public void testCanExitModuleEnd() {
 		    	 request.loopCount=0;
 		    result = "";
 		       request.exitMethod="exitTemplate"
-		          module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		          bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		         result &= "body";
 		       }
 		       result &= "aftermodule";
@@ -241,7 +241,7 @@ public void testCanExitModuleEnd() {
 		    request.loopCount=0;
 		    result = "";
 		    request.exitMethod="loop"
-		    module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
+		    bx:module template="src/test/java/ortus/boxlang/runtime/components/system/ExitTests/module.bxm" {
 		    	result &= "body";
 		    }
 		    result &= "aftermodule";
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java
index 767fc23a5..386c147a1 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/InvokeTest.java
@@ -99,7 +99,7 @@ public void testInvokeCurrentContext() {
 				function foo() {
 					return "bar";
 				}
-				invoke method="foo" returnVariable="result";
+				bx:invoke method="foo" returnVariable="result";
 		    """,
 		    context );
 		// @formatter:on
@@ -167,9 +167,9 @@ function foo() {
 	public void testInvokeExistingClass() {
 		instance.executeSource(
 		    """
-		    myClass = new src.test.java.ortus.boxlang.runtime.components.system.InvokeTest()
-		         invoke class="#myClass#" method="foo" returnVariable="result" ;
-		         """,
+		       myClass = new src.test.java.ortus.boxlang.runtime.components.system.InvokeTest()
+		    bx:invoke class="#myClass#" method="foo" returnVariable="result" ;
+		            """,
 		    context );
 		assertThat( variables.get( result ) ).isEqualTo( "bar" );
 	}
@@ -179,8 +179,8 @@ public void testInvokeExistingClass() {
 	public void testInvokeCreatedClass() {
 		instance.executeSource(
 		    """
-		    	 invoke class="src.test.java.ortus.boxlang.runtime.components.system.InvokeTest" method="foo" returnVariable="result" ;
-		    """,
+		    bx:invoke class="src.test.java.ortus.boxlang.runtime.components.system.InvokeTest" method="foo" returnVariable="result" ;
+		      """,
 		    context );
 		assertThat( variables.get( result ) ).isEqualTo( "bar" );
 	}
@@ -190,8 +190,8 @@ public void testInvokeCreatedClass() {
 	public void testTranspile() {
 		instance.executeSource(
 		    """
-		    	 invoke component="src.test.java.ortus.boxlang.runtime.components.system.InvokeTest" method="foo" returnVariable="result" ;
-		    """,
+		    invoke component="src.test.java.ortus.boxlang.runtime.components.system.InvokeTest" method="foo" returnVariable="result" ;
+		      """,
 		    context, BoxSourceType.CFSCRIPT );
 		assertThat( variables.get( result ) ).isEqualTo( "bar" );
 	}
@@ -206,7 +206,7 @@ public void testInvokeOnAStruct() {
 		        		return "bar";
 		        	}
 		        }
-		    	invoke class="#myStr#" method="foo" returnVariable="result" ;
+		    	bx:invoke class="#myStr#" method="foo" returnVariable="result" ;
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.get( result ) ).isEqualTo( "bar" );
@@ -306,7 +306,7 @@ function meh( x=3 ) {
 		    			 variables.result = arguments;
 		    		 }
 		    createArgs()
-		    		 invoke method="meh" argumentCollection="#args#";
+		    		 bx:invoke method="meh" argumentCollection="#args#";
 		    	 """,
 		    context );
 
@@ -330,7 +330,7 @@ function meh( a ) {
 		    		variables.result = arguments;
 		    	}
 		    	createArgs('hello world')
-		       		 invoke method="meh" argumentCollection="#args#";
+		       		 bx:invoke method="meh" argumentCollection="#args#";
 		    """,
 		    context );
 
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LogTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LogTest.java
index 1731fffe3..039ee8001 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/LogTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/LogTest.java
@@ -85,7 +85,7 @@ public void setupEach() {
 	public void testComponentScript() {
 		instance.executeSource(
 		    """
-		    log text="Hello Logger!" file="bxlog";
+		    bx:log text="Hello Logger!" file="bxlog";
 		    """,
 		    context, BoxSourceType.BOXSCRIPT );
 		assertTrue( StringUtils.contains( outContent.toString(), "Hello Logger!" ) );
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java
index 01e6910ca..bd73f2a5e 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/LoopTest.java
@@ -238,11 +238,11 @@ public void testcfloopQueryGrouped() {
 	public void testLoopTimes() {
 		instance.executeSource(
 		    """
-		    result = "";
-		    	loop times=5 {
-		    		result &= "*";
-		    	}
-		          """,
+		       result = "";
+		    bx:loop times=5 {
+		       		result &= "*";
+		       	}
+		             """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "*****" );
@@ -252,11 +252,11 @@ public void testLoopTimes() {
 	public void testLoopZeroTimes() {
 		instance.executeSource(
 		    """
-		    result = "";
-		    	loop times=0 {
-		    		result &= "*";
-		    	}
-		          """,
+		       result = "";
+		    bx:loop times=0 {
+		       		result &= "*";
+		       	}
+		             """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "" );
@@ -266,11 +266,11 @@ public void testLoopZeroTimes() {
 	public void testLoopTimesIndex() {
 		instance.executeSource(
 		    """
-		    result = "";
-		    	loop times=5 index="i" {
-		    		result &= i;
-		    	}
-		          """,
+		       result = "";
+		    bx:loop times=5 index="i" {
+		       		result &= i;
+		       	}
+		             """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "12345" );
@@ -280,11 +280,11 @@ public void testLoopTimesIndex() {
 	public void testLoopTimesItem() {
 		instance.executeSource(
 		    """
-		    result = "";
-		    	loop times=5 item="i" {
-		    		result &= i;
-		    	}
-		          """,
+		       result = "";
+		    bx:loop times=5 item="i" {
+		       		result &= i;
+		       	}
+		             """,
 		    context, BoxSourceType.BOXSCRIPT );
 
 		assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "12345" );
@@ -391,15 +391,15 @@ 
 	public void testLoopCondition() {
 		instance.executeSource(
 		    """
-		     function foo( required string name ) {
-		    	 loop condition=arguments.name == "brad" {
-		    return getFunctionCalledName();
-		    		 break;
-		    	 }
-		     }
-
-		     result = foo( "brad" );
-		    				 """,
+		       function foo( required string name ) {
+		    bx:loop condition=arguments.name == "brad" {
+		      return getFunctionCalledName();
+		      		 break;
+		      	 }
+		       }
+
+		       result = foo( "brad" );
+		      				 """,
 		    context, BoxSourceType.BOXSCRIPT );
 		assertThat( variables.getAsString( Key.of( "result" ) ) ).isEqualTo( "foo" );
 	}
@@ -409,7 +409,7 @@ public void testLoopToFrom() {
 		instance.executeSource(
 		    """
 		    	result = ""
-		    	loop from="1" to="5" step="1" index="i" {
+		    	bx:loop from="1" to="5" step="1" index="i" {
 		    		result &= i;
 		    	}
 		    """,
@@ -422,7 +422,7 @@ public void testLoopToFromNegativeStep() {
 		instance.executeSource(
 		    """
 		    	result = ""
-		    	loop from="5" to="1" step="-1" index="i" {
+		    	bx:loop from="5" to="1" step="-1" index="i" {
 		    		result &= i;
 		    	}
 		    """,
@@ -435,7 +435,7 @@ public void testLoopToFromZeroStep() {
 		instance.executeSource(
 		    """
 		    	result = ""
-		    	loop from="1" to="5" step="0" index="i" {
+		    	bx:loop from="1" to="5" step="0" index="i" {
 		    		result &= i;
 		    	}
 		    """,
@@ -448,7 +448,7 @@ public void testLoopToFromDecimalStep() {
 		instance.executeSource(
 		    """
 		    	result = ""
-		    	loop from="1" to="10" step="1.5" index="i" {
+		    	bx:loop from="1" to="10" step="1.5" index="i" {
 		    		result = result.listAppend(i)
 		    	}
 		    """,
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/ModuleTest.java b/src/test/java/ortus/boxlang/runtime/components/system/ModuleTest.java
index ccd54d5d6..a90c4679d 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/ModuleTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/ModuleTest.java
@@ -18,7 +18,14 @@
 
 package ortus.boxlang.runtime.components.system;
 
-import org.junit.jupiter.api.*;
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import ortus.boxlang.compiler.parser.BoxSourceType;
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.context.IBoxContext;
@@ -27,8 +34,6 @@
 import ortus.boxlang.runtime.scopes.Key;
 import ortus.boxlang.runtime.scopes.VariablesScope;
 
-import static com.google.common.truth.Truth.assertThat;
-
 public class ModuleTest {
 
 	static BoxRuntime	instance;
@@ -92,9 +97,9 @@ public void testCanRunCustomTagScript() {
 
 		instance.executeSource(
 		    """
-		    brad="wood";
-		       module template="src/test/java/ortus/boxlang/runtime/components/system/MyTag.cfm" foo="bar";
-		              """,
+		       brad="wood";
+		    bx:module template="src/test/java/ortus/boxlang/runtime/components/system/MyTag.cfm" foo="bar";
+		                 """,
 		    context );
 		assertThat( buffer.toString().replaceAll( "\\s", "" ) ).isEqualTo( "alwaysMyTagstartbarwood" );
 		assertThat( variables.getAsString( result ) ).isEqualTo( "hey you guys" );
diff --git a/src/test/java/ortus/boxlang/runtime/components/system/SaveContentTest.java b/src/test/java/ortus/boxlang/runtime/components/system/SaveContentTest.java
index a7b550df3..fccbfb107 100644
--- a/src/test/java/ortus/boxlang/runtime/components/system/SaveContentTest.java
+++ b/src/test/java/ortus/boxlang/runtime/components/system/SaveContentTest.java
@@ -64,7 +64,7 @@ public void testCanCaptureContentScript() {
 		instance.executeSource(
 		    """
 		       	echo( "before" );
-		          saveContent variable="result" {
+		          bx:saveContent variable="result" {
 		       	echo( "Hello World" );
 		       }
 		    echo( "after" );

From c2223cf017d823ecaf8b72d02f53fa4f7f0e452c Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Wed, 15 Jan 2025 15:14:23 +0100
Subject: [PATCH 144/161] oops

---
 coldbox.sh | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100755 coldbox.sh

diff --git a/coldbox.sh b/coldbox.sh
deleted file mode 100755
index e85559925..000000000
--- a/coldbox.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-cd ~/Sites/projects/boxlang
-git pull --rebase --autostash
-gradle build -x test -x javadoc
-
-cd ~/Sites/projects/boxlang-web-support
-git pull --rebase --autostash
-gradle build -x test -x javadoc
-
-cd ~/Sites/projects/boxlang-servlet
-git pull --rebase --autostash
-gradle buildRuntime

From c02727192ef3f7c23db090f4d211bdae6f165e7d Mon Sep 17 00:00:00 2001
From: Michael Born 
Date: Wed, 15 Jan 2025 10:00:53 -0500
Subject: [PATCH 145/161] JDBC - Fix for queries with duplicate column names

---
 .../java/ortus/boxlang/runtime/types/Query.java  |  4 ++--
 .../bifs/global/jdbc/QueryExecuteTest.java       | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java
index 66a8cb14f..3cc85013c 100644
--- a/src/main/java/ortus/boxlang/runtime/types/Query.java
+++ b/src/main/java/ortus/boxlang/runtime/types/Query.java
@@ -162,9 +162,9 @@ public static Query fromResultSet( ResultSet resultSet ) {
 			}
 
 			while ( resultSet.next() ) {
-				Object[] row = new Object[ columnCount ];
+				IStruct row = new Struct( IStruct.TYPES.LINKED );
 				for ( int i = 1; i <= columnCount; i++ ) {
-					row[ i - 1 ] = resultSet.getObject( i );
+					row.put( resultSetMetaData.getColumnLabel( i ), resultSet.getObject( resultSetMetaData.getColumnLabel( i ) ) );
 				}
 				query.addRow( row );
 			}
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java
index e6fb938b7..66fd6634b 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/jdbc/QueryExecuteTest.java
@@ -779,6 +779,22 @@ public void testCacheResultMeta() {
 		assertEquals( Duration.ofMinutes( 30 ), result.get( Key.cacheLastAccessTimeout ) );
 	}
 
+	@DisplayName( "It can properly handle duplicate column names in the result set" )
+	@Test
+	public void testDuplicateColumnResultSets() {
+		instance.executeStatement(
+		    """
+		        result = queryExecute( "SELECT name, CURRENT_DATE AS name, id, role FROM developers WHERE id=1" );
+		    """, context );
+		assertThat( variables.get( result ) ).isInstanceOf( Query.class );
+		Query query = variables.getAsQuery( result );
+		assertEquals( 1, query.size() );
+		IStruct firstRow = query.getRowAsStruct( 0 );
+		assertThat( firstRow.get( Key._NAME ) ).isEqualTo( "Luis Majano" );
+		assertThat( firstRow.get( Key.id ) ).isEqualTo( 1 );
+		assertThat( firstRow.get( Key.of( "role" ) ) ).isEqualTo( "CEO" );
+	}
+
 	@Disabled( "Not implemented" )
 	@DisplayName( "It only keeps the first resultSet and discards the rest like Lucee" )
 	@Test

From 75715e0a59348fcedccc67f07eaf7e10161e63d4 Mon Sep 17 00:00:00 2001
From: Michael Born 
Date: Wed, 15 Jan 2025 10:13:21 -0500
Subject: [PATCH 146/161] JDBC - Tweak query row insertion to not overwrite
 nulls as empty strings

Use containsKey to check for the keys existence before assuming we should set it to an empty string.
---
 src/main/java/ortus/boxlang/runtime/types/Query.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java
index 3cc85013c..739880883 100644
--- a/src/main/java/ortus/boxlang/runtime/types/Query.java
+++ b/src/main/java/ortus/boxlang/runtime/types/Query.java
@@ -164,7 +164,8 @@ public static Query fromResultSet( ResultSet resultSet ) {
 			while ( resultSet.next() ) {
 				IStruct row = new Struct( IStruct.TYPES.LINKED );
 				for ( int i = 1; i <= columnCount; i++ ) {
-					row.put( resultSetMetaData.getColumnLabel( i ), resultSet.getObject( resultSetMetaData.getColumnLabel( i ) ) );
+					String columnName = resultSetMetaData.getColumnLabel( i );
+					row.put( columnName, resultSet.getObject( columnName ) );
 				}
 				query.addRow( row );
 			}
@@ -539,7 +540,7 @@ public int addRow( IStruct row ) {
 		Object		o;
 		for ( QueryColumn column : columns.values() ) {
 			// Missing keys in the struct go in the query as an empty string (CF compat)
-			rowData[ i ] = ( o = row.get( column.getName() ) ) == null ? "" : o;
+			rowData[ i ] = row.containsKey( column.getName() ) ? row.get( column.getName() ) : "";
 			i++;
 		}
 		// We're ignoring extra keys in the struct that aren't query columns. Lucee

From 3d2d108e5b930c9010caa779a6c2c16463c59520 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Wed, 15 Jan 2025 14:18:54 -0600
Subject: [PATCH 147/161] BL-905 Update approach

---
 .../ortus/boxlang/runtime/types/Query.java    | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/Query.java b/src/main/java/ortus/boxlang/runtime/types/Query.java
index 739880883..637d3e654 100644
--- a/src/main/java/ortus/boxlang/runtime/types/Query.java
+++ b/src/main/java/ortus/boxlang/runtime/types/Query.java
@@ -153,19 +153,32 @@ public static Query fromResultSet( ResultSet resultSet ) {
 		try {
 			ResultSetMetaData	resultSetMetaData	= resultSet.getMetaData();
 			int					columnCount			= resultSetMetaData.getColumnCount();
+			// This will map which column in the JDBC result corresponds with the ordinal position of each query column
+			List		columnMapList		= new ArrayList<>();
 
 			// The column count starts from 1
 			for ( int i = 1; i <= columnCount; i++ ) {
-				query.addColumn(
-				    Key.of( resultSetMetaData.getColumnLabel( i ) ),
-				    QueryColumnType.fromSQLType( resultSetMetaData.getColumnType( i ) ) );
+				Key colName = Key.of( resultSetMetaData.getColumnLabel( i ) );
+				// If we haven't hit this column name before....
+				if ( !query.hasColumn( colName ) ) {
+					// Add it
+					query.addColumn(
+					    colName,
+					    QueryColumnType.fromSQLType( resultSetMetaData.getColumnType( i ) ) );
+					// And remember this col possition as where the data will come from
+					columnMapList.add( i );
+				}
 			}
 
+			// Native array for super fast access
+			int[] columnMap = columnMapList.stream().mapToInt( i -> i ).toArray();
+			// Update, may be smaller now if there were duplicate column names
+			columnCount = columnMap.length;
 			while ( resultSet.next() ) {
-				IStruct row = new Struct( IStruct.TYPES.LINKED );
-				for ( int i = 1; i <= columnCount; i++ ) {
-					String columnName = resultSetMetaData.getColumnLabel( i );
-					row.put( columnName, resultSet.getObject( columnName ) );
+				Object[] row = new Object[ columnCount ];
+				for ( int i = 0; i < columnCount; i++ ) {
+					// Get the data in the JDBC column based on our column map
+					row[ i ] = resultSet.getObject( columnMap[ i ] );
 				}
 				query.addRow( row );
 			}

From 4582f8f5bf79bc0f55a7df20b89267ea4fdce6a3 Mon Sep 17 00:00:00 2001
From: Jon Clausen 
Date: Thu, 16 Jan 2025 11:48:09 -0500
Subject: [PATCH 148/161] BL-940 Resolve

---
 .../java/ortus/boxlang/runtime/events/InterceptorPool.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java b/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
index 952e99af2..7338702a7 100644
--- a/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
+++ b/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
@@ -38,6 +38,7 @@
 import ortus.boxlang.runtime.types.IStruct;
 import ortus.boxlang.runtime.types.Struct;
 import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException;
+import ortus.boxlang.runtime.types.exceptions.AbortException;
 
 /**
  * An InterceptorPool is a pool of interceptors that can be used to intercept events
@@ -591,15 +592,15 @@ public void announce( Key state, IStruct data ) {
 	public void announce( Key state, IStruct data, IBoxContext context ) {
 		if ( hasState( state ) ) {
 			// logger.trace( "InterceptorService.announce() - announcing {}", state.getName() );
-
 			try {
 				getState( state ).announce( data, context );
+			} catch ( AbortException e ) {
+				throw e;
 			} catch ( Exception e ) {
 				String errorMessage = String.format( "Errors announcing [%s] interception", state.getName() );
 				logger.error( errorMessage, e );
 				throw new BoxRuntimeException( errorMessage, e );
 			}
-
 			// logger.trace( "Finished announcing {}", state.getName() );
 		} else {
 			// logger.trace( "InterceptorService.announce() - No state found for: {}", state.getName() );

From 449e6152b3bbb4e62216f294a90ae7d64bdfb643 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Thu, 16 Jan 2025 19:15:47 +0100
Subject: [PATCH 149/161] BL-942 #resolve

Interceptor Service now does a service loader load of all interceptors found in the runtime to auto load them

BL-941 related
---
 .../ortus/boxlang/runtime/BoxRuntime.java     | 17 -------
 .../boxlang/runtime/events/Interceptor.java   | 42 +++++++++++++++++
 .../runtime/interceptors/ASTCapture.java      | 18 ++++++--
 .../boxlang/runtime/interceptors/Logging.java |  9 ++++
 .../runtime/interop/proxies/BaseProxy.java    | 10 +++-
 .../interop/proxies/IInterceptorLambda.java   |  6 +++
 .../boxlang/runtime/modules/ModuleRecord.java |  3 ++
 .../runtime/services/InterceptorService.java  | 46 +++++++++++++++++++
 8 files changed, 129 insertions(+), 22 deletions(-)
 create mode 100644 src/main/java/ortus/boxlang/runtime/events/Interceptor.java

diff --git a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
index e8195907e..2a84576ba 100644
--- a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
+++ b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
@@ -53,12 +53,9 @@
 import ortus.boxlang.runtime.context.RequestBoxContext;
 import ortus.boxlang.runtime.context.RuntimeBoxContext;
 import ortus.boxlang.runtime.context.ScriptingRequestBoxContext;
-import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
 import ortus.boxlang.runtime.dynamic.casters.CastAttempt;
 import ortus.boxlang.runtime.dynamic.casters.StringCaster;
 import ortus.boxlang.runtime.events.BoxEvent;
-import ortus.boxlang.runtime.interceptors.ASTCapture;
-import ortus.boxlang.runtime.interceptors.Logging;
 import ortus.boxlang.runtime.interop.DynamicObject;
 import ortus.boxlang.runtime.loader.ClassLocator;
 import ortus.boxlang.runtime.loader.DynamicClassLoader;
@@ -366,20 +363,6 @@ private void loadConfiguration( Boolean debugMode, String configPath ) {
 
 		// Reconfigure the logging services
 		this.loggingService.reconfigure();
-
-		// AST Capture experimental feature
-		BooleanCaster.attempt(
-		    this.configuration.experimental.getOrDefault( "ASTCapture", false ) ).ifSuccessful(
-		        astCapture -> {
-			        if ( astCapture ) {
-				        this.interceptorService.register(
-				            DynamicObject.of( new ASTCapture( false, true ) ),
-				            Key.onParse );
-			        }
-		        } );
-
-		// Load core logger and other core interceptions
-		this.interceptorService.register( new Logging( this ) );
 	}
 
 	/**
diff --git a/src/main/java/ortus/boxlang/runtime/events/Interceptor.java b/src/main/java/ortus/boxlang/runtime/events/Interceptor.java
new file mode 100644
index 000000000..f0ede2490
--- /dev/null
+++ b/src/main/java/ortus/boxlang/runtime/events/Interceptor.java
@@ -0,0 +1,42 @@
+/**
+ * [BoxLang]
+ *
+ * Copyright [2023] [Ortus Solutions, Corp]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ortus.boxlang.runtime.events;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation that can be used on an Interceptor to alter it's
+ * registration/loading and runtime behavior
+ */
+@Documented
+@Retention( RetentionPolicy.RUNTIME )
+@Target( ElementType.TYPE )
+public @interface Interceptor {
+
+	/**
+	 * Indicates whether the interception should be automatically loaded.
+	 * Default is true.
+	 *
+	 * @return true if it should be auto-loaded, false otherwise.
+	 */
+	boolean autoLoad() default true;
+}
diff --git a/src/main/java/ortus/boxlang/runtime/interceptors/ASTCapture.java b/src/main/java/ortus/boxlang/runtime/interceptors/ASTCapture.java
index 712b6b20f..af45301db 100644
--- a/src/main/java/ortus/boxlang/runtime/interceptors/ASTCapture.java
+++ b/src/main/java/ortus/boxlang/runtime/interceptors/ASTCapture.java
@@ -17,17 +17,23 @@
  */
 package ortus.boxlang.runtime.interceptors;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+
 import ortus.boxlang.compiler.parser.ParsingResult;
 import ortus.boxlang.runtime.events.BaseInterceptor;
 import ortus.boxlang.runtime.events.InterceptionPoint;
+import ortus.boxlang.runtime.events.Interceptor;
 import ortus.boxlang.runtime.types.IStruct;
 
-import java.io.IOException;
-import java.nio.file.*;
-
 /**
  * An interceptor that captures the AST and outputs it to the console or a file
  */
+@Interceptor( autoLoad = false )
 public class ASTCapture extends BaseInterceptor {
 
 	private boolean		toConsole	= false;
@@ -35,6 +41,12 @@ public class ASTCapture extends BaseInterceptor {
 	// Default to current working directory
 	private final Path	filePath	= Paths.get( "./grapher/data/" );
 
+	/**
+	 * No-arg Constructor
+	 */
+	public ASTCapture() {
+	}
+
 	/**
 	 * Constructor
 	 *
diff --git a/src/main/java/ortus/boxlang/runtime/interceptors/Logging.java b/src/main/java/ortus/boxlang/runtime/interceptors/Logging.java
index b035397d5..517a49ea6 100644
--- a/src/main/java/ortus/boxlang/runtime/interceptors/Logging.java
+++ b/src/main/java/ortus/boxlang/runtime/interceptors/Logging.java
@@ -20,6 +20,7 @@
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.events.BaseInterceptor;
 import ortus.boxlang.runtime.events.InterceptionPoint;
+import ortus.boxlang.runtime.events.Interceptor;
 import ortus.boxlang.runtime.logging.LoggingService;
 import ortus.boxlang.runtime.scopes.Key;
 import ortus.boxlang.runtime.types.IStruct;
@@ -27,6 +28,7 @@
 /**
  * A BoxLang interceptor that provides logging capabilities
  */
+@Interceptor( autoLoad = true )
 public class Logging extends BaseInterceptor {
 
 	private static final String	DEFAULT_LOGGER	= "application";
@@ -34,6 +36,13 @@ public class Logging extends BaseInterceptor {
 	private LoggingService		loggingService;
 	private BoxRuntime			runtime;
 
+	/**
+	 * No-Arg Constructor
+	 */
+	public Logging() {
+		this( BoxRuntime.getInstance() );
+	}
+
 	/**
 	 * Constructor
 	 *
diff --git a/src/main/java/ortus/boxlang/runtime/interop/proxies/BaseProxy.java b/src/main/java/ortus/boxlang/runtime/interop/proxies/BaseProxy.java
index 15eea50f8..985e7ba30 100644
--- a/src/main/java/ortus/boxlang/runtime/interop/proxies/BaseProxy.java
+++ b/src/main/java/ortus/boxlang/runtime/interop/proxies/BaseProxy.java
@@ -76,6 +76,12 @@ public abstract class BaseProxy {
 	    Key.wait
 	);
 
+	/**
+	 * No-arg constructor for the proxy.
+	 */
+	protected BaseProxy() {
+	}
+
 	/**
 	 * Constructor for the proxy.
 	 *
@@ -133,7 +139,7 @@ protected BaseProxy( Object target, IBoxContext context ) {
 	 * @param args    The arguments to pass to the function
 	 *
 	 * @return The result of the function
-	 * 
+	 *
 	 * @throws InterruptedException
 	 */
 	protected Object invoke( Key method, Object... args ) throws InterruptedException {
@@ -186,7 +192,7 @@ protected Object invoke( Key method, Object... args ) throws InterruptedExceptio
 	 * @param args The arguments to pass to the function
 	 *
 	 * @return The result of the function
-	 * 
+	 *
 	 * @throws InterruptedException
 	 */
 	protected Object invoke( Object... args ) throws InterruptedException {
diff --git a/src/main/java/ortus/boxlang/runtime/interop/proxies/IInterceptorLambda.java b/src/main/java/ortus/boxlang/runtime/interop/proxies/IInterceptorLambda.java
index abbb513c3..46b73e86b 100644
--- a/src/main/java/ortus/boxlang/runtime/interop/proxies/IInterceptorLambda.java
+++ b/src/main/java/ortus/boxlang/runtime/interop/proxies/IInterceptorLambda.java
@@ -19,11 +19,17 @@
 
 import ortus.boxlang.runtime.context.IBoxContext;
 import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
+import ortus.boxlang.runtime.events.Interceptor;
 import ortus.boxlang.runtime.types.IStruct;
 import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException;
 
+@Interceptor( autoLoad = false )
 public class IInterceptorLambda extends BaseProxy implements ortus.boxlang.runtime.events.IInterceptorLambda {
 
+	public IInterceptorLambda() {
+		super();
+	}
+
 	public IInterceptorLambda( Object target, IBoxContext context, String method ) {
 		super( target, context, method );
 		prepLogger( IInterceptorLambda.class );
diff --git a/src/main/java/ortus/boxlang/runtime/modules/ModuleRecord.java b/src/main/java/ortus/boxlang/runtime/modules/ModuleRecord.java
index 62428fda4..552c0b980 100644
--- a/src/main/java/ortus/boxlang/runtime/modules/ModuleRecord.java
+++ b/src/main/java/ortus/boxlang/runtime/modules/ModuleRecord.java
@@ -518,6 +518,9 @@ public ModuleRecord register( IBoxContext context ) {
 		// Do we have any Java IInterceptor to register in the InterceptorService
 		ServiceLoader.load( IInterceptor.class, this.classLoader )
 		    .stream()
+		    // Only load interceptors that are set to auto-load by default or by configuration
+		    .filter( provider -> interceptorService.canLoadInterceptor( provider.type() ) )
+		    // Register the interceptor
 		    .map( ServiceLoader.Provider::get )
 		    .forEach( interceptorService::register );
 
diff --git a/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java b/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
index 9343ffaa8..fe9731f96 100644
--- a/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
+++ b/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
@@ -17,11 +17,18 @@
  */
 package ortus.boxlang.runtime.services;
 
+import java.util.ServiceLoader;
+
 import org.slf4j.Logger;
 
 import ortus.boxlang.runtime.BoxRuntime;
+import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
 import ortus.boxlang.runtime.events.BoxEvent;
+import ortus.boxlang.runtime.events.IInterceptor;
+import ortus.boxlang.runtime.events.Interceptor;
 import ortus.boxlang.runtime.events.InterceptorPool;
+import ortus.boxlang.runtime.interceptors.ASTCapture;
+import ortus.boxlang.runtime.interop.DynamicObject;
 import ortus.boxlang.runtime.scopes.Key;
 
 /**
@@ -75,6 +82,45 @@ public InterceptorService( BoxRuntime runtime ) {
 	@Override
 	public void onConfigurationLoad() {
 		this.logger = runtime.getLoggingService().getLogger( "runtime" );
+
+		// AST Capture experimental feature
+		BooleanCaster.attempt(
+		    this.runtime.getConfiguration().experimental.getOrDefault( "ASTCapture", false ) )
+		    .ifSuccessful(
+		        astCapture -> {
+			        if ( astCapture ) {
+				        register( DynamicObject.of( new ASTCapture( false, true ) ), Key.onParse );
+			        }
+		        } );
+
+		// Auto-Load all interceptors found in the runtime classloader
+		ServiceLoader.load( IInterceptor.class, this.runtime.getRuntimeLoader() )
+		    .stream()
+		    // Only load interceptors that are set to auto-load by default or by configuration
+		    .filter( provider -> canLoadInterceptor( provider.type() ) )
+		    // Register the interceptor
+		    .map( ServiceLoader.Provider::get )
+		    .forEach( this::register );
+	}
+
+	/**
+	 * This method encapsulates the logic to determine if an interceptor can be loaded or not.
+	 *
+	 * @param targetClass The class of the interceptor to be loaded
+	 *
+	 * @return True if the interceptor can be loaded, false otherwise
+	 */
+	public boolean canLoadInterceptor( Class targetClass ) {
+		// Check the @Interceptor annotation config properties
+		// AutoLoad defaults to true if the annotation is not found.
+		if ( targetClass.isAnnotationPresent( Interceptor.class ) ) {
+			Interceptor annotation = targetClass.getAnnotation( Interceptor.class );
+			if ( !annotation.autoLoad() ) {
+				logger.debug( "Interceptor [{}] is set to not auto-load, skipping.", targetClass.getName() );
+				return false;
+			}
+		}
+		return true;
 	}
 
 	/**

From c132bb29e7c538d2aefd142b7edaba51c38ae808 Mon Sep 17 00:00:00 2001
From: Michael Born 
Date: Thu, 16 Jan 2025 14:05:34 -0500
Subject: [PATCH 150/161] JDBC - Add connection string to connection failure
 error message

This should make debugging much easier!
---
 .../java/ortus/boxlang/runtime/jdbc/DataSource.java    | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java b/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java
index d8bacd7db..b209a116a 100644
--- a/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java
+++ b/src/main/java/ortus/boxlang/runtime/jdbc/DataSource.java
@@ -19,6 +19,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
 
 import ortus.boxlang.runtime.BoxRuntime;
@@ -77,10 +78,15 @@ public DataSource( DatasourceConfig config ) {
 		);
 		// Retrieve and store the potentially modified configuration from the event.
 		this.configuration = eventParams.getAs( DatasourceConfig.class, Key.of( "config" ) );
+		HikariConfig hikariConfig = null;
 		try {
-			this.hikariDataSource = new HikariDataSource( this.configuration.toHikariConfig() );
+			hikariConfig			= this.configuration.toHikariConfig();
+			this.hikariDataSource	= new HikariDataSource( hikariConfig );
 		} catch ( RuntimeException e ) {
-			throw new BoxRuntimeException( "Unable to create datasource connection: " + e.getMessage(), e );
+			String message = hikariConfig != null
+			    ? "Unable to create datasource connection to URL [" + hikariConfig.getJdbcUrl() + "] : "
+			    : "Unable to create datasource connection: ";
+			throw new BoxRuntimeException( message + e.getMessage(), e );
 		}
 	}
 

From 22d566c21782118a0bc6d7a96e46a06dde5e65c5 Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Thu, 16 Jan 2025 16:15:38 -0600
Subject: [PATCH 151/161] Removing dead code

---
 src/main/java/ortus/boxlang/runtime/types/Function.java | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/Function.java b/src/main/java/ortus/boxlang/runtime/types/Function.java
index 1ae6e36c8..3aa4223c8 100644
--- a/src/main/java/ortus/boxlang/runtime/types/Function.java
+++ b/src/main/java/ortus/boxlang/runtime/types/Function.java
@@ -220,21 +220,14 @@ public Object invoke( FunctionBoxContext context ) {
 	 * @return
 	 */
 	protected Object ensureReturnType( IBoxContext context, Object value ) {
-		// CF doesn't enforce return types on null returns. I think that is a bug, but we'd need a compat layer to make existing CF code work.
 		if ( value == null ) {
 			return null;
 		}
 		CastAttempt typeCheck = GenericCaster.attempt( context, value, getReturnType(), true );
 		if ( !typeCheck.wasSuccessful() ) {
-			String actualType;
-			if ( value == null ) {
-				actualType = "null";
-			} else {
-				actualType = value.getClass().getName();
-			}
 			throw new BoxRuntimeException(
 			    String.format( "The return value of the function [%s] is of type [%s] does not match the declared type of [%s]",
-			        getName().getName(), actualType, getReturnType() )
+			        getName().getName(), value.getClass().getName(), getReturnType() )
 			);
 		}
 		if ( typeCheck.get() instanceof NullValue ) {

From b101df6b182ef3b1e6e7f33b709833ed46000e0d Mon Sep 17 00:00:00 2001
From: Brad Wood 
Date: Thu, 16 Jan 2025 16:15:53 -0600
Subject: [PATCH 152/161] This is passing, just a typo

---
 src/test/java/TestCases/phase3/ClassTest.java     | 1 -
 src/test/java/TestCases/phase3/RelativeManager.bx | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java
index 4831a73db..d83547d2b 100644
--- a/src/test/java/TestCases/phase3/ClassTest.java
+++ b/src/test/java/TestCases/phase3/ClassTest.java
@@ -1351,7 +1351,6 @@ public void testRelativeInstantiation() {
 		assertThat( variables.getAsStruct( Key.of( "result2" ) ).getAsString( Key._NAME ) ).isEqualTo( "src.test.java.TestCases.phase3.FindMe" );
 	}
 
-	@Disabled( "Brad Fix this pretty please!!" )
 	@Test
 	public void testRelativeInstantiationWithLoop() {
 		// @formatter:off
diff --git a/src/test/java/TestCases/phase3/RelativeManager.bx b/src/test/java/TestCases/phase3/RelativeManager.bx
index 393a1af89..31c03af50 100644
--- a/src/test/java/TestCases/phase3/RelativeManager.bx
+++ b/src/test/java/TestCases/phase3/RelativeManager.bx
@@ -5,7 +5,7 @@ class{
 		return newFuture().allApply( argumentCollection = arguments );
 	}
 
-	RelativeInstantiation function newFuture(){
+	Future function newFuture(){
 		return new tasks.Future();
 	}
 

From 5b2de907a821299e078a1b6490e118d55c27ced6 Mon Sep 17 00:00:00 2001
From: Jon Clausen 
Date: Thu, 16 Jan 2025 17:46:55 -0500
Subject: [PATCH 153/161] BL-943 Resolve - Ensure returned XMLElemNew objects
 are XML types

---
 .../runtime/bifs/global/xml/XMLElemNew.java   |  6 +--
 .../java/ortus/boxlang/runtime/types/XML.java |  6 +--
 .../types/listeners/XMLChildrenListener.java  |  6 +--
 .../bifs/global/xml/XMLElemNewTest.java       | 37 ++++++++++++++++++-
 4 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNew.java b/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNew.java
index b8535b352..e477330a1 100644
--- a/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNew.java
+++ b/src/main/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNew.java
@@ -85,7 +85,7 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
 					xmlString = "<" + childName + "/>";
 				}
 
-				return builder.parse( new InputSource( new StringReader( xmlString ) ) );
+				return new XML( builder.parse( new InputSource( new StringReader( xmlString ) ) ) );
 
 			} catch ( ParserConfigurationException e ) {
 				throw new BoxRuntimeException( "Error creating XML parser", e );
@@ -96,10 +96,10 @@ public Object _invoke( IBoxContext context, ArgumentsScope arguments ) {
 			}
 		} else if ( namespace != null ) {
 			Document ownerDocument = documentNode.getOwnerDocument() == null ? ( Document ) documentNode : documentNode.getOwnerDocument();
-			return ownerDocument.createElementNS( namespace, childName );
+			return new XML( ownerDocument.createElementNS( namespace, childName ) );
 		} else {
 			Document ownerDocument = documentNode.getOwnerDocument() == null ? ( Document ) documentNode : documentNode.getOwnerDocument();
-			return ownerDocument.createElement( childName );
+			return new XML( ownerDocument.createElement( childName ) );
 		}
 
 	}
diff --git a/src/main/java/ortus/boxlang/runtime/types/XML.java b/src/main/java/ortus/boxlang/runtime/types/XML.java
index dd36cfb6c..dc5097432 100644
--- a/src/main/java/ortus/boxlang/runtime/types/XML.java
+++ b/src/main/java/ortus/boxlang/runtime/types/XML.java
@@ -660,13 +660,13 @@ public boolean containsValue( Object value ) {
 
 	@Override
 	public Object put( Key key, Object value ) {
-		if ( ! ( value instanceof Node ) ) {
+		if ( ! ( value instanceof Node ) && ! ( value instanceof XML ) ) {
 			throw new BoxRuntimeException(
 			    String.format(
-			        "The value passed [%s] is not a Node instance", value.toString() )
+			        "The value passed [%s] is not a Node or XML instance", value.toString() )
 			);
 		}
-		value = ( ( Node ) value ).cloneNode( true );
+		value = value instanceof Node ? ( ( Node ) value ).cloneNode( true ) : ( ( XML ) value ).getNode().cloneNode( true );
 		if ( documentOnlyKeys.contains( key ) ) {
 			if ( key.equals( Key.XMLRoot ) ) {
 				this.node = ( Node ) value;
diff --git a/src/main/java/ortus/boxlang/runtime/types/listeners/XMLChildrenListener.java b/src/main/java/ortus/boxlang/runtime/types/listeners/XMLChildrenListener.java
index 29ba9347e..4b25f0692 100644
--- a/src/main/java/ortus/boxlang/runtime/types/listeners/XMLChildrenListener.java
+++ b/src/main/java/ortus/boxlang/runtime/types/listeners/XMLChildrenListener.java
@@ -53,12 +53,12 @@ public Object notify( Key key, Object newValue, Object oldValue ) {
 			parentNode.removeChild( childNodeList.item( index ) );
 		} else if ( newValue != null && oldValue == null ) {
 			if ( childNodeList.item( index ) == null ) {
-				parentNode.appendChild( ( Node ) newValue );
+				parentNode.appendChild( newValue instanceof Node ? ( Node ) newValue : ( ( XML ) newValue ).getNode() );
 			} else {
-				parentNode.insertBefore( ( Node ) newValue, childNodeList.item( index ) );
+				parentNode.insertBefore( newValue instanceof Node ? ( Node ) newValue : ( ( XML ) newValue ).getNode(), childNodeList.item( index ) );
 			}
 		} else if ( newValue == null && oldValue != null ) {
-			parentNode.removeChild( ( Node ) oldValue );
+			parentNode.removeChild( oldValue instanceof Node ? ( Node ) oldValue : ( ( XML ) oldValue ).getNode() );
 		}
 		return null;
 	}
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
index 7c9e76775..bff07dfab 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
@@ -71,7 +71,7 @@ public void testBif() {
 		    myXML.xmlRoot.xmlChildren.append( xmlElemNew( myXML, "myChild" ) ).append( xmlElemNew( myXML, "myChild" ) ).prepend( xmlElemNew( myXML, "mySecondChild" ) );
 		          """,
 		    context );
-		assertTrue( variables.get( result ) instanceof Node );
+		assertTrue( variables.get( result ) instanceof XML );
 		XML doc = variables.getAsXML( Key.of( "myXML" ) );
 		assertEquals( "root", doc.getNode().getFirstChild().getNodeName() );
 		assertEquals( 3, doc.getNode().getFirstChild().getChildNodes().getLength() );
@@ -88,7 +88,7 @@ public void testBifNamespace() {
 		    myXML.xmlRoot = result;
 		       """,
 		    context );
-		assertTrue( variables.get( result ) instanceof Node );
+		assertTrue( variables.get( result ) instanceof XML );
 		XML doc = variables.getAsXML( Key.of( "myXML" ) );
 		assertEquals( "Envelope", doc.getNode().getFirstChild().getNodeName() );
 		NamedNodeMap attributes = doc.getNode().getFirstChild().getAttributes();
@@ -96,4 +96,37 @@ public void testBifNamespace() {
 
 	}
 
+	@DisplayName( "It tests the ability to append the result of XMLElemNew to the XML children" )
+	@Test
+	public void testNodeAppend() {
+		instance.executeSource(
+		    """
+		    xmlObj = xmlParse( '' );
+		    newNode = xmlElemNew( xmlObj.xmlRoot, "BoxLang" );
+		    xmlObj.xmlRoot.xmlChildren.append( newNode );
+		    result = xmlObj.xmlRoot.BoxLang.xmlName;
+		    childCount = xmlObj.xmlRoot.xmlChildren.len();
+		         """,
+		    context );
+		assertTrue( variables.get( result ) instanceof String );
+		assertEquals( "BoxLang", variables.getAsString( result ) );
+		assertEquals( 1, variables.getAsInteger( Key.of( "childCount" ) ) );
+
+	}
+
+	@DisplayName( "It tests the ability to use the return XMLElemNew as a node" )
+	@Test
+	public void testNodeReturn() {
+		instance.executeSource(
+		    """
+		    xmlObj = xmlParse( '' );
+		    newNode = xmlElemNew( xmlObj.xmlRoot, "BoxLang" );
+		    result = newNode.xmlName;
+		         """,
+		    context );
+		assertTrue( variables.get( result ) instanceof String );
+		assertEquals( "BoxLang", variables.getAsString( result ) );
+
+	}
+
 }

From 6dfc9d00642b937597b20b38f0622160a7e0e645 Mon Sep 17 00:00:00 2001
From: Jon Clausen 
Date: Thu, 16 Jan 2025 19:08:46 -0500
Subject: [PATCH 154/161] BL-944 Resolve - fix direct assignment of XMLText and
 CDATA

---
 .../java/ortus/boxlang/runtime/types/XML.java | 16 +++++++++++-
 .../ortus/boxlang/runtime/types/XMLTest.java  | 26 +++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/XML.java b/src/main/java/ortus/boxlang/runtime/types/XML.java
index dc5097432..0ddcdbc81 100644
--- a/src/main/java/ortus/boxlang/runtime/types/XML.java
+++ b/src/main/java/ortus/boxlang/runtime/types/XML.java
@@ -57,6 +57,7 @@
 import ortus.boxlang.runtime.dynamic.casters.CastAttempt;
 import ortus.boxlang.runtime.dynamic.casters.KeyCaster;
 import ortus.boxlang.runtime.dynamic.casters.NumberCaster;
+import ortus.boxlang.runtime.dynamic.casters.StringCaster;
 import ortus.boxlang.runtime.interop.DynamicInteropService;
 import ortus.boxlang.runtime.scopes.IntKey;
 import ortus.boxlang.runtime.scopes.Key;
@@ -109,6 +110,12 @@ public class XML implements Serializable, IStruct {
 	 */
 	private static final long		serialVersionUID	= 1L;
 
+	/**
+	 * CDATA contstants
+	 */
+	public static final String		cdataStart			= "";
+
 	/**
 	 * Create a new XML Document from the given string
 	 */
@@ -542,7 +549,14 @@ public Object dereferenceAndInvoke( IBoxContext context, Key name, Map' );
+		    xmlObj.xmlRoot.xmlText = "BoxLang";
+		    result = xmlObj.xmlRoot.xmlText;
+		         """,
+		    context );
+		assertThat( variables.get( result ) ).isEqualTo( "BoxLang" );
+	}
+
+	@DisplayName( "It can assign XMLCDATA" )
+	@Test
+	void testAssignXMLCDATA() {
+		instance.executeSource(
+		    """
+		    xmlObj = xmlParse( '' );
+		    xmlObj.xmlRoot.xmlCDATA = "";
+		    result = xmlObj.xmlRoot.xmlCDATA;
+		         """,
+		    context );
+		assertThat( variables.get( result ) ).isEqualTo( "]]>" );
+	}
+
 }

From 9a022ca51d6ac8082eba46755802a04e1410b788 Mon Sep 17 00:00:00 2001
From: Jon Clausen 
Date: Thu, 16 Jan 2025 20:42:45 -0500
Subject: [PATCH 155/161] BL-945 Resolve - implement attributes modifications
 for XML objects

---
 .../java/ortus/boxlang/runtime/types/XML.java | 14 +++-
 .../bifs/global/xml/XMLElemNewTest.java       | 84 +++++++++++++++++++
 .../ortus/boxlang/runtime/types/XMLTest.java  | 28 +++++++
 3 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/types/XML.java b/src/main/java/ortus/boxlang/runtime/types/XML.java
index 0ddcdbc81..effe32015 100644
--- a/src/main/java/ortus/boxlang/runtime/types/XML.java
+++ b/src/main/java/ortus/boxlang/runtime/types/XML.java
@@ -50,6 +50,8 @@
 import org.w3c.dom.ProcessingInstruction;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
+import org.w3c.dom.Attr;
+import org.w3c.dom.Element;
 
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.bifs.MemberDescriptor;
@@ -243,13 +245,21 @@ public String getNodeComments() {
 	 * @return the attributes of this XML node as a struct
 	 */
 	public IStruct getXMLAttributes() {
-		// TODO: attach change listener to the struct so changes in the struct will be reflected in the XML
-		IStruct			attributes	= new Struct( IStruct.TYPES.LINKED );
+		Struct			attributes	= new Struct( IStruct.TYPES.LINKED );
 		NamedNodeMap	attrs		= node.getAttributes();
 		for ( int i = 0; i < attrs.getLength(); i++ ) {
 			Node attr = attrs.item( i );
 			attributes.put( Key.of( attr.getNodeName() ), attr.getNodeValue() );
 		}
+		attributes.registerChangeListener( ( key, newValue, oldValue ) -> {
+			if ( newValue == null ) {
+				node.getAttributes().removeNamedItem( key.getName() );
+			} else {
+				( ( Element ) node ).setAttribute( key.getName(), StringCaster.cast( newValue ) );
+			}
+			return newValue;
+		} );
+
 		return attributes;
 	}
 
diff --git a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
index bff07dfab..6dec31dad 100644
--- a/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
+++ b/src/test/java/ortus/boxlang/runtime/bifs/global/xml/XMLElemNewTest.java
@@ -25,6 +25,7 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.w3c.dom.NamedNodeMap;
@@ -129,4 +130,87 @@ public void testNodeReturn() {
 
 	}
 
+	@DisplayName( "Tests some typing and abstraction of XML objects" )
+	@Test
+	@Disabled
+	void testTyping() {
+		//@formatter:off
+		instance.executeSource(
+		    """
+			xml function createXMLNode(
+				required xml parent,
+				required string name,
+				any value,
+				struct attributes = {},
+				struct children   = {}
+			){
+				var newNode = xmlElemNew( arguments.parent, arguments.name );
+				if ( !isNull( arguments.value ) && isSimpleValue( arguments.value ) ) {
+					newNode.xmlText = arguments.value;
+				} else if ( !isNull( arguments.value ) && isStruct( arguments.value ) ) {
+					arguments.attributes = arguments.value;
+				}
+				arguments.attributes
+					.keyArray()
+					.each( function( key ){
+						if ( !isNull( attributes[ key ] ) ) {
+							newNode.xmlAttributes[ key ] = attributes[ key ];
+						}
+					} );
+				if ( !arguments.children.isEmpty() ) {
+					appendChildNodes( parent = newNode, children = arguments.children );
+				}
+				arrayAppend( parent.xmlChildren, newNode );
+				return parent.xmlChildren[ parent.xmlChildren.len() ];
+			}
+
+			void function appendChildNodes( required xml parent, required struct children ){
+				arguments.children
+					.keyArray()
+					.filter( ( childName ) => !isNull( children[ childName ] ) )
+					.each( function( childName ){
+						if ( !isStruct( children[ childName ] ) ) {
+							createXMLNode(
+								parent = parent,
+								name   = childName,
+								value  = children[ childName ]
+							);
+						} else {
+							var nodeArgs         = children[ childName ];
+							nodeArgs[ "parent" ] = parent;
+							nodeArgs[ "name" ]   = childName;
+							createXMLNode( argumentCollection = nodeArgs );
+						}
+					} );
+			}
+
+			xmlObj = xmlParse( "" );
+			createXMLNode( xmlObj.xmlRoot, "BoxLang", "Rocks!" );
+			// createXMLNode( 
+			// 	xmlObj.xmlRoot.BoxLang,
+			// 	{
+			// 		"Developers" : {
+			// 			"Majano" : {
+			// 				attributes : {
+			// 					"firstName" : "Luis",
+			// 					"lastName" : "Majano"
+			// 				}
+			// 			}
+			// 		}
+
+			// 	},
+			// 	{
+			// 		"tagline" : "Boxlang is good, better, best"
+			// 	}
+			// );
+				printLine( toString( xmlObj ) );
+		         """,
+		    context );
+		// @formatter:on
+		// assertTrue( variables.get( result ) instanceof String );
+		// assertTrue( variables.get( result ) instanceof String );
+		// assertEquals( "BoxLang", variables.getAsString( result ) );
+		// assertEquals( "Ortus", variables.getAsString( result ) );
+	}
+
 }
diff --git a/src/test/java/ortus/boxlang/runtime/types/XMLTest.java b/src/test/java/ortus/boxlang/runtime/types/XMLTest.java
index 9ae1d1b62..d3a2dd6a4 100644
--- a/src/test/java/ortus/boxlang/runtime/types/XMLTest.java
+++ b/src/test/java/ortus/boxlang/runtime/types/XMLTest.java
@@ -312,4 +312,32 @@ void testAssignXMLCDATA() {
 		assertThat( variables.get( result ) ).isEqualTo( "]]>" );
 	}
 
+	@DisplayName( "It can assign an XML Atrribute" )
+	@Test
+	void testAssignXMLAttributes() {
+		instance.executeSource(
+		    """
+		    xmlObj = xmlParse( '' );
+		    xmlObj.xmlRoot.xmlAttributes.Product = "BoxLang";
+		    result = xmlObj.xmlRoot.xmlAttributes;
+		         """,
+		    context );
+		assertThat( variables.get( result ) ).isInstanceOf( Struct.class );
+		assertThat( variables.getAsStruct( result ).get( "Product" ) ).isEqualTo( "BoxLang" );
+	}
+
+	@DisplayName( "It can remove an XML Atrribute" )
+	@Test
+	void testRemoveXMLAttributes() {
+		instance.executeSource(
+		    """
+		       xmlObj = xmlParse( '' );
+		    structDelete( xmlObj.xmlRoot.xmlAttributes, "Product" );
+		       result = xmlObj.xmlRoot.xmlAttributes;
+		            """,
+		    context );
+		assertThat( variables.get( result ) ).isInstanceOf( Struct.class );
+		assertThat( variables.getAsStruct( result ).get( "Product" ) ).isEqualTo( null );
+	}
+
 }

From 2ce12e85997742c230b1c9af7c0d1c405c04f962 Mon Sep 17 00:00:00 2001
From: Jacob Beers 
Date: Thu, 16 Jan 2025 21:34:23 -0600
Subject: [PATCH 156/161] BL-946 Populate the return value of getEnclosingClass

---
 .../compiler/asmboxpiler/AsmTranspiler.java      | 14 ++++++++------
 src/test/java/TestCases/phase3/ClassTest.java    | 16 ++++++++++++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java
index f5d6ce106..164a55a72 100644
--- a/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java
+++ b/src/main/java/ortus/boxlang/compiler/asmboxpiler/AsmTranspiler.java
@@ -441,12 +441,14 @@ public ClassNode transpile( BoxScript boxScript ) throws BoxRuntimeException {
 		Type type = Type.getType( "L" + getProperty( "packageName" ).replace( '.', '/' ) + "/" + getProperty( "classname" ) + ";" );
 		setProperty( "classType", type.getDescriptor() );
 		setProperty( "classTypeInternal", type.getInternalName() );
-		ClassNode	classNode		= new ClassNode();
-		String		mappingName		= getProperty( "mappingName" );
-		String		mappingPath		= getProperty( "mappingPath" );
-		String		relativePath	= getProperty( "relativePath" );
-		Source		source			= boxScript.getPosition().getSource();
-		String		filePath		= source instanceof SourceFile file && file.getFile() != null ? file.getFile().getAbsolutePath() : "unknown";
+		ClassNode classNode = new ClassNode();
+		setOwningClass( classNode );
+		setProperty( "enclosingClassInternalName", type.getInternalName() );
+		String	mappingName		= getProperty( "mappingName" );
+		String	mappingPath		= getProperty( "mappingPath" );
+		String	relativePath	= getProperty( "relativePath" );
+		Source	source			= boxScript.getPosition().getSource();
+		String	filePath		= source instanceof SourceFile file && file.getFile() != null ? file.getFile().getAbsolutePath() : "unknown";
 		setProperty( "filePath", filePath );
 		classNode.visitSource( filePath, null );
 
diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java
index d83547d2b..735621f8a 100644
--- a/src/test/java/TestCases/phase3/ClassTest.java
+++ b/src/test/java/TestCases/phase3/ClassTest.java
@@ -1575,6 +1575,22 @@ public void testUDFClassEnclosingClassReference() {
 		assertThat( variables.get( "outerClass" ) ).isEqualTo( variables.get( "innerClassesOuterClass" ) );
 	}
 
+	@DisplayName( "udf class has enclosing class reference" )
+	@Test
+	public void testUDFClassEnclosingClassReferenceInTemplate() {
+
+		instance.executeSource(
+		    """
+		    function test(){
+
+		     }
+		         result = test.getClass().getEnclosingClass();
+		                """,
+		    context );
+
+		assertThat( variables.get( result ) ).isNotNull();
+	}
+
 	@DisplayName( "mixins should be public" )
 	@Test
 	public void testMixinsPublic() {

From 815dd497d437f01b69cb6f0a5ef4f99f595b9b96 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Thu, 16 Jan 2025 22:37:33 +0100
Subject: [PATCH 157/161] hmm fixed tests but not sure why they work

---
 src/test/java/TestCases/phase3/ClassTest.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/TestCases/phase3/ClassTest.java b/src/test/java/TestCases/phase3/ClassTest.java
index 735621f8a..ee549281a 100644
--- a/src/test/java/TestCases/phase3/ClassTest.java
+++ b/src/test/java/TestCases/phase3/ClassTest.java
@@ -1351,6 +1351,7 @@ public void testRelativeInstantiation() {
 		assertThat( variables.getAsStruct( Key.of( "result2" ) ).getAsString( Key._NAME ) ).isEqualTo( "src.test.java.TestCases.phase3.FindMe" );
 	}
 
+	// @Disabled( "Brad Fix this pretty please!!" )
 	@Test
 	public void testRelativeInstantiationWithLoop() {
 		// @formatter:off

From 1f425590350d5a40f79b647671631520e65a18bb Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Fri, 17 Jan 2025 14:25:35 +0100
Subject: [PATCH 158/161] logging encapsulation to runtime for interceptors

---
 .../runtime/events/InterceptorPool.java       | 40 ++++++++++++++-----
 .../runtime/services/InterceptorService.java  | 16 ++++----
 2 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java b/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
index 7338702a7..9ffc0567a 100644
--- a/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
+++ b/src/main/java/ortus/boxlang/runtime/events/InterceptorPool.java
@@ -25,20 +25,18 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.context.IBoxContext;
 import ortus.boxlang.runtime.interop.DynamicObject;
 import ortus.boxlang.runtime.loader.ClassLocator.ClassLocation;
+import ortus.boxlang.runtime.logging.BoxLangLogger;
 import ortus.boxlang.runtime.modules.ModuleRecord;
 import ortus.boxlang.runtime.runnables.IClassRunnable;
 import ortus.boxlang.runtime.scopes.Key;
 import ortus.boxlang.runtime.types.IStruct;
 import ortus.boxlang.runtime.types.Struct;
-import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException;
 import ortus.boxlang.runtime.types.exceptions.AbortException;
+import ortus.boxlang.runtime.types.exceptions.BoxRuntimeException;
 
 /**
  * An InterceptorPool is a pool of interceptors that can be used to intercept events
@@ -60,7 +58,7 @@ public class InterceptorPool {
 	/**
 	 * Logger
 	 */
-	private static final Logger					logger				= LoggerFactory.getLogger( InterceptorPool.class );
+	private BoxLangLogger						logger;
 
 	/**
 	 * The list of interception points we can listen for
@@ -330,7 +328,7 @@ public IClassRunnable newAndRegister(
 		// - ModuleRecord : The ModuleRecord instance, if passed
 		oInterceptor.getVariablesScope().put( Key._NAME, name );
 		oInterceptor.getVariablesScope().put( Key.properties, properties );
-		oInterceptor.getVariablesScope().put( Key.log, LoggerFactory.getLogger( oInterceptor.getClass() ) );
+		oInterceptor.getVariablesScope().put( Key.log, getLogger() );
 		oInterceptor.getVariablesScope().put( Key.interceptorService, this );
 		oInterceptor.getVariablesScope().put( Key.boxRuntime, this.runtime );
 
@@ -477,6 +475,15 @@ public InterceptorPool register( IClassRunnable interceptor ) {
 	 * @return The same pool
 	 */
 	public InterceptorPool register( DynamicObject interceptor, Key... states ) {
+
+		if ( getLogger().isTraceEnabled() ) {
+			getLogger().trace(
+			    "=> Registering interceptor [{}] with states: {}",
+			    interceptor.getClass().getName(),
+			    states
+			);
+		}
+
 		Arrays.stream( states )
 		    .forEach( state -> registerState( state ).register( interceptor ) );
 		return this;
@@ -591,19 +598,17 @@ public void announce( Key state, IStruct data ) {
 	 */
 	public void announce( Key state, IStruct data, IBoxContext context ) {
 		if ( hasState( state ) ) {
-			// logger.trace( "InterceptorService.announce() - announcing {}", state.getName() );
 			try {
 				getState( state ).announce( data, context );
 			} catch ( AbortException e ) {
 				throw e;
 			} catch ( Exception e ) {
 				String errorMessage = String.format( "Errors announcing [%s] interception", state.getName() );
-				logger.error( errorMessage, e );
+				getLogger().error( errorMessage, e );
 				throw new BoxRuntimeException( errorMessage, e );
 			}
-			// logger.trace( "Finished announcing {}", state.getName() );
 		} else {
-			// logger.trace( "InterceptorService.announce() - No state found for: {}", state.getName() );
+			getLogger().trace( "InterceptorService.announce() - No state found for: {}", state.getName() );
 		}
 	}
 
@@ -687,7 +692,7 @@ public CompletableFuture announceAsync( Key state, IStruct data, IBoxCo
 					return data;
 				} catch ( Exception e ) {
 					String errorMessage = String.format( "Errors announcing [%s] interception", state.getName() );
-					logger.error( errorMessage, e );
+					getLogger().error( errorMessage, e );
 					throw new BoxRuntimeException( errorMessage, e );
 				}
 			} );
@@ -696,4 +701,17 @@ public CompletableFuture announceAsync( Key state, IStruct data, IBoxCo
 		return null;
 	}
 
+	/**
+	 * Get the runtime logger for interceptors
+	 */
+	public BoxLangLogger getLogger() {
+		if ( this.logger == null ) {
+			synchronized ( this ) {
+				if ( this.logger == null ) {
+					this.logger = this.runtime.getLoggingService().getLogger( "runtime" );
+				}
+			}
+		}
+		return this.logger;
+	}
 }
diff --git a/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java b/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
index fe9731f96..7da4a5536 100644
--- a/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
+++ b/src/main/java/ortus/boxlang/runtime/services/InterceptorService.java
@@ -19,8 +19,6 @@
 
 import java.util.ServiceLoader;
 
-import org.slf4j.Logger;
-
 import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.dynamic.casters.BooleanCaster;
 import ortus.boxlang.runtime.events.BoxEvent;
@@ -29,6 +27,7 @@
 import ortus.boxlang.runtime.events.InterceptorPool;
 import ortus.boxlang.runtime.interceptors.ASTCapture;
 import ortus.boxlang.runtime.interop.DynamicObject;
+import ortus.boxlang.runtime.logging.BoxLangLogger;
 import ortus.boxlang.runtime.scopes.Key;
 
 /**
@@ -50,9 +49,9 @@ public class InterceptorService extends InterceptorPool implements IService {
 	 */
 
 	/**
-	 * Logger
+	 * The interceptor logger goes into the `runtime` category
 	 */
-	private Logger logger;
+	private BoxLangLogger logger;
 
 	/**
 	 * --------------------------------------------------------------------------
@@ -81,7 +80,8 @@ public InterceptorService( BoxRuntime runtime ) {
 	 */
 	@Override
 	public void onConfigurationLoad() {
-		this.logger = runtime.getLoggingService().getLogger( "runtime" );
+		// Startup the logger
+		getLogger();
 
 		// AST Capture experimental feature
 		BooleanCaster.attempt(
@@ -116,7 +116,7 @@ public boolean canLoadInterceptor( Class targetClass ) {
 		if ( targetClass.isAnnotationPresent( Interceptor.class ) ) {
 			Interceptor annotation = targetClass.getAnnotation( Interceptor.class );
 			if ( !annotation.autoLoad() ) {
-				logger.debug( "Interceptor [{}] is set to not auto-load, skipping.", targetClass.getName() );
+				getLogger().debug( "Interceptor [{}] is set to not auto-load, skipping.", targetClass.getName() );
 				return false;
 			}
 		}
@@ -128,7 +128,7 @@ public boolean canLoadInterceptor( Class targetClass ) {
 	 */
 	@Override
 	public void onStartup() {
-		logger.debug( "InterceptorService.onStartup()" );
+		getLogger().debug( "InterceptorService.onStartup()" );
 	}
 
 	/**
@@ -138,7 +138,7 @@ public void onStartup() {
 	 */
 	@Override
 	public void onShutdown( Boolean force ) {
-		logger.debug( "InterceptorService.onShutdown()" );
+		getLogger().debug( "InterceptorService.onShutdown()" );
 	}
 
 }

From d0e93b1450009d4e342b7e5f8a7a96ec1af7dc52 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Fri, 17 Jan 2025 15:21:33 +0100
Subject: [PATCH 159/161] - fixing tests I broke

---
 .../boxlang/runtime/cache/providers/BoxCacheProviderTest.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java b/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
index 437704ebd..1c07cb35e 100644
--- a/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
+++ b/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
@@ -26,6 +26,7 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+import ortus.boxlang.runtime.BoxRuntime;
 import ortus.boxlang.runtime.async.executors.ExecutorRecord;
 import ortus.boxlang.runtime.cache.filters.WildcardFilter;
 import ortus.boxlang.runtime.config.segments.CacheConfig;
@@ -37,6 +38,7 @@
 
 public class BoxCacheProviderTest {
 
+	static BoxRuntime		instance;
 	static BoxCacheProvider	boxCache;
 	static CacheConfig		config			= new CacheConfig();
 	static CacheService		cacheService	= Mockito.mock( CacheService.class );
@@ -44,6 +46,7 @@ public class BoxCacheProviderTest {
 
 	@BeforeAll
 	static void setup() {
+		instance = BoxRuntime.getInstance( true );
 		Mockito.when( cacheService.getTaskScheduler() ).thenReturn( executorRecord );
 		Mockito.when( cacheService.getRuntime() ).thenReturn( Mockito.mock( ortus.boxlang.runtime.BoxRuntime.class ) );
 

From 0e22581c122a13a78dbc36f5671951f61bc33da8 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Fri, 17 Jan 2025 15:24:37 +0100
Subject: [PATCH 160/161] fixed tests again

---
 .../boxlang/runtime/cache/providers/BoxCacheProviderTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java b/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
index 1c07cb35e..63a3220fa 100644
--- a/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
+++ b/src/test/java/ortus/boxlang/runtime/cache/providers/BoxCacheProviderTest.java
@@ -48,7 +48,7 @@ public class BoxCacheProviderTest {
 	static void setup() {
 		instance = BoxRuntime.getInstance( true );
 		Mockito.when( cacheService.getTaskScheduler() ).thenReturn( executorRecord );
-		Mockito.when( cacheService.getRuntime() ).thenReturn( Mockito.mock( ortus.boxlang.runtime.BoxRuntime.class ) );
+		Mockito.when( cacheService.getRuntime() ).thenReturn( instance );
 
 		boxCache = new BoxCacheProvider();
 		boxCache.configure( cacheService, config );

From 24f3343fccd5077f0682c25f23bc96e0fc6208c7 Mon Sep 17 00:00:00 2001
From: Luis Majano 
Date: Fri, 17 Jan 2025 15:59:39 +0100
Subject: [PATCH 161/161] chicken and egg issue

---
 .../ortus/boxlang/runtime/BoxRuntime.java     | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
index 2a84576ba..714a22d1d 100644
--- a/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
+++ b/src/main/java/ortus/boxlang/runtime/BoxRuntime.java
@@ -469,6 +469,16 @@ private void startup( Boolean debugMode ) {
 		loadConfiguration( debugMode, this.configPath );
 		// Anythying below might use configuration items
 
+		// Ensure home assets
+		ensureHomeAssets();
+
+		// Load the Dynamic Class Loader for the runtime
+		this.runtimeLoader = new DynamicClassLoader(
+		    Key.runtime,
+		    getConfiguration().getJavaLibraryPaths(),
+		    this.getClass().getClassLoader(),
+		    true );
+
 		// Announce it to the services
 		this.interceptorService.onConfigurationLoad();
 		this.asyncService.onConfigurationLoad();
@@ -480,17 +490,8 @@ private void startup( Boolean debugMode ) {
 		this.schedulerService.onConfigurationLoad();
 		this.dataSourceService.onConfigurationLoad();
 
-		// Ensure home assets
-		ensureHomeAssets();
-
-		// Load the Dynamic Class Loader for the runtime
-		this.runtimeLoader	= new DynamicClassLoader(
-		    Key.runtime,
-		    getConfiguration().getJavaLibraryPaths(),
-		    this.getClass().getClassLoader(),
-		    true );
 		// Startup the right Compiler
-		this.boxpiler		= chooseBoxpiler();
+		this.boxpiler = chooseBoxpiler();
 		// Seed Mathematical Precision for the runtime
 		MathUtil.setHighPrecisionMath( getConfiguration().useHighPrecisionMath );