From 931de2768f223f83a934f3a4bffb310619c49a93 Mon Sep 17 00:00:00 2001 From: Joshua Lochner Date: Sat, 16 Dec 2023 17:14:13 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20[jinja]=20Add=20support=20for=20?= =?UTF-8?q?evaluating=20logical=20expressions=20between=20non-Booleans=20(?= =?UTF-8?q?#420)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds support for computing logical expressions between non-Booleans, by considering their truthy nature. Needed for [this](https://discuss.huggingface.co/t/issue-with-llama-2-chat-template-and-out-of-date-documentation/61645/3) complicated chat template. (cc @Rocketknight1) --- packages/jinja/src/runtime.ts | 55 +++-- packages/jinja/test/templates.test.js | 308 +++++++++++++++++++++++++- 2 files changed, 341 insertions(+), 22 deletions(-) diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index b2a57995e..6bc3646f2 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -46,6 +46,15 @@ abstract class RuntimeValue { constructor(value: T = undefined as unknown as T) { this.value = value; } + + /** + * Determines truthiness or falsiness of the runtime value. + * This function should be overridden by subclasses if it has custom truthiness criteria. + * @returns {BooleanValue} BooleanValue(true) if the value is truthy, BooleanValue(false) otherwise. + */ + __bool__(): BooleanValue { + return new BooleanValue(!!this.value); + } } /** @@ -96,6 +105,18 @@ export class BooleanValue extends RuntimeValue { */ export class ObjectValue extends RuntimeValue> { override type = "ObjectValue"; + + /** + * NOTE: necessary to override since all JavaScript arrays are considered truthy, + * while only non-empty Python arrays are consider truthy. + * + * e.g., + * - JavaScript: {} && 5 -> 5 + * - Python: {} and 5 -> {} + */ + override __bool__(): BooleanValue { + return new BooleanValue(this.value.size > 0); + } } /** @@ -104,6 +125,18 @@ export class ObjectValue extends RuntimeValue> { export class ArrayValue extends RuntimeValue { override type = "ArrayValue"; override builtins = new Map([["length", new NumericValue(this.value.length)]]); + + /** + * NOTE: necessary to override since all JavaScript arrays are considered truthy, + * while only non-empty Python arrays are consider truthy. + * + * e.g., + * - JavaScript: [] && 5 -> 5 + * - Python: [] and 5 -> [] + */ + override __bool__(): BooleanValue { + return new BooleanValue(this.value.length > 0); + } } /** @@ -218,12 +251,19 @@ export class Interpreter { const left = this.evaluate(node.left, environment); const right = this.evaluate(node.right, environment); - // Arbitrary equality comparison + // Arbitrary operands switch (node.operator.value) { + // Equality operators case "==": return new BooleanValue(left.value == right.value); case "!=": return new BooleanValue(left.value != right.value); + + // Logical operators + case "and": + return left.__bool__().value ? right : left; + case "or": + return left.__bool__().value ? left : right; } if (left instanceof UndefinedValue || right instanceof UndefinedValue) { @@ -255,14 +295,6 @@ export class Interpreter { case "<=": return new BooleanValue(left.value <= right.value); } - } else if (left instanceof BooleanValue && right instanceof BooleanValue) { - // Logical operators - switch (node.operator.value) { - case "and": - return new BooleanValue(left.value && right.value); - case "or": - return new BooleanValue(left.value || right.value); - } } else if (right instanceof ArrayValue) { const member = right.value.find((x) => x.value === left.value) !== undefined; switch (node.operator.value) { @@ -480,10 +512,7 @@ export class Interpreter { private evaluateIf(node: If, environment: Environment): StringValue { const test = this.evaluate(node.test, environment); - if (!["BooleanValue", "BooleanLiteral"].includes(test.type)) { - throw new Error(`Expected boolean expression in if statement: got ${test.type}`); - } - return this.evaluateBlock(test.value ? node.body : node.alternate, environment); + return this.evaluateBlock(test.__bool__().value ? node.body : node.alternate, environment); } private evaluateFor(node: For, environment: Environment): StringValue { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index d543582f7..728cb0167 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -69,6 +69,13 @@ const TEST_STRINGS = { // Filter operator FILTER_OPERATOR: `{{ arr | length }}{{ 1 + arr | length }}{{ 2 + arr | sort | length }}{{ (arr | sort)[0] }}`, + + // Logical operators between non-Booleans + BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`, + BOOLEAN_STRINGS: `|{{ 'a' and 'b' }}|{{ 'a' and '' }}|{{ '' and 'a' }}|{{ '' and '' }}|{{ 'a' or 'b' }}|{{ 'a' or '' }}|{{ '' or 'a' }}|{{ '' or '' }}|{{ not 'a' }}|{{ not '' }}|`, + BOOLEAN_MIXED: `|{{ true and 1 }}|{{ true and 0 }}|{{ false and 1 }}|{{ false and 0 }}|{{ true or 1 }}|{{ true or 0 }}|{{ false or 1 }}|{{ false or 0 }}|`, + BOOLEAN_MIXED_2: `|{{ true and '' }}|{{ true and 'a' }}|{{ false or '' }}|{{ false or 'a' }}|{{ '' and true }}|{{ 'a' and true }}|{{ '' or false }}|{{ 'a' or false }}|`, + BOOLEAN_MIXED_IF: `{% if '' %}{{ 'A' }}{% endif %}{% if 'a' %}{{ 'B' }}{% endif %}{% if true and '' %}{{ 'C' }}{% endif %}{% if true and 'a' %}{{ 'D' }}{% endif %}`, }; const TEST_PARSED = { @@ -1132,6 +1139,278 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, ], + + // Logical operators between non-Booleans + BOOLEAN_NUMERICAL: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "and", type: "And" }, + { value: "2", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "and", type: "And" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "and", type: "And" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "and", type: "And" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "or", type: "Or" }, + { value: "2", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "or", type: "Or" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "or", type: "Or" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "or", type: "Or" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "UnaryOperator" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "UnaryOperator" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + BOOLEAN_STRINGS: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "b", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "a", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "b", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "a", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "UnaryOperator" }, + { value: "a", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "UnaryOperator" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + BOOLEAN_MIXED: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + BOOLEAN_MIXED_2: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "a", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Or" }, + { value: "a", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "true", type: "BooleanLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "and", type: "And" }, + { value: "true", type: "BooleanLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "false", type: "BooleanLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "or", type: "Or" }, + { value: "false", type: "BooleanLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + BOOLEAN_MIXED_IF: [ + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "If" }, + { value: "", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "A", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "EndIf" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "If" }, + { value: "a", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "B", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "EndIf" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "If" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "C", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "EndIf" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "If" }, + { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "And" }, + { value: "a", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "D", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "EndIf" }, + { value: "%}", type: "CloseStatement" }, + ], }; const TEST_CONTEXT = { @@ -1239,6 +1518,13 @@ const TEST_CONTEXT = { FILTER_OPERATOR: { arr: [3, 2, 1], }, + + // Logical operators between non-Booleans + BOOLEAN_NUMERICAL: {}, + BOOLEAN_STRINGS: {}, + BOOLEAN_MIXED: {}, + BOOLEAN_MIXED_2: {}, + BOOLEAN_MIXED_IF: {}, }; const EXPECTED_OUTPUTS = { @@ -1308,6 +1594,13 @@ const EXPECTED_OUTPUTS = { // Filter operator FILTER_OPERATOR: `3451`, + + // Logical operators between non-Booleans + BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`, + BOOLEAN_STRINGS: `|b||||a|a|a||false|true|`, + BOOLEAN_MIXED: `|1|0|false|false|true|true|1|0|`, + BOOLEAN_MIXED_2: `||a||a||true|false|a|`, + BOOLEAN_MIXED_IF: `BD`, }; describe("Templates", () => { @@ -1419,6 +1712,12 @@ describe("Error checking", () => { const tokens = tokenize(text); expect(() => parse(tokens)).toThrowError(); }); + + it("Invalid control structure usage", () => { + const text = "{% if %}Content{% endif %}"; + const tokens = tokenize(text); + expect(() => parse(tokens)).toThrowError(); + }); }); describe("Runtime errors", () => { @@ -1466,15 +1765,6 @@ describe("Error checking", () => { expect(() => interpreter.run(ast)).toThrowError(); }); - it("Invalid control structure usage", () => { - const env = new Environment(); - const interpreter = new Interpreter(env); - - const tokens = tokenize("{% if 42 %}Content{% endif %}"); - const ast = parse(tokens); - expect(() => interpreter.run(ast)).toThrowError(); - }); - it("Invalid variable assignment", () => { const env = new Environment(); const interpreter = new Interpreter(env);