Skip to content

Commit

Permalink
[jinja] Add more built-in functions (#484)
Browse files Browse the repository at this point in the history 
This PR adds support for several built-in jinja filters (from [this
list](https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters)).

Related discussion:
https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf/discussions/21
  • Loading branch information
@xenova
xenova committed Feb 15, 2024
1 parent d57fc81 commit 65b4dad
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 3 deletions.
30 changes: 27 additions & 3 deletions packages/jinja/src/runtime.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,7 @@ export class Interpreter {
// }
// return filter.value([operand], environment);

// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters
if (operand instanceof ArrayValue) {
switch (node.filter.value) {
case "first":
Expand Down Expand Up @@ -368,12 +369,35 @@ export class Interpreter {
})
);
default:
throw new Error(`Unknown filter: ${node.filter.value}`);
throw new Error(`Unknown ArrayValue filter: ${node.filter.value}`);
}
} else if (operand instanceof StringValue) {
switch (node.filter.value) {
case "length":
return new NumericValue(operand.value.length);
case "upper":
return new StringValue(operand.value.toUpperCase());
case "lower":
return new StringValue(operand.value.toLowerCase());
case "title":
return new StringValue(operand.value.replace(/\b\w/g, (c) => c.toUpperCase()));
case "capitalize":
return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
case "trim":
return new StringValue(operand.value.trim());
default:
throw new Error(`Unknown StringValue filter: ${node.filter.value}`);
}
} else if (operand instanceof NumericValue) {
switch (node.filter.value) {
case "abs":
return new NumericValue(Math.abs(operand.value));
default:
throw new Error(`Unknown NumericValue filter: ${node.filter.value}`);
}
}

// TODO add support for StringValue operand
throw new Error(`Cannot apply filter to type: ${operand.type}`);
throw new Error(`Cannot apply filter "${node.filter.value}" to type: ${operand.type}`);
}

/**
Expand Down
70 changes: 70 additions & 0 deletions packages/jinja/test/templates.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ const TEST_STRINGS = {

// Filter operator
FILTER_OPERATOR: `{{ arr | length }}{{ 1 + arr | length }}{{ 2 + arr | sort | length }}{{ (arr | sort)[0] }}`,
FILTER_OPERATOR_2: `|{{ 'abc' | length }}|{{ 'aBcD' | upper }}|{{ 'aBcD' | lower }}|{{ 'test test' | capitalize}}|{{ 'test test' | title }}|{{ ' a b ' | trim }}|{{ ' A B ' | trim | lower | length }}|`,
FILTER_OPERATOR_3: `|{{ -1 | abs }}|{{ 1 | abs }}|`,

// Logical operators between non-Booleans
BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`,
Expand Down Expand Up @@ -1139,6 +1141,70 @@ const TEST_PARSED = {
{ value: "]", type: "CloseSquareBracket" },
{ value: "}}", type: "CloseExpression" },
],
FILTER_OPERATOR_2: [
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "abc", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "length", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "aBcD", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "upper", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "aBcD", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "lower", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "test test", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "capitalize", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "test test", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "title", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: " a b ", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "trim", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: " A B ", type: "StringLiteral" },
{ value: "|", type: "Pipe" },
{ value: "trim", type: "Identifier" },
{ value: "|", type: "Pipe" },
{ value: "lower", type: "Identifier" },
{ value: "|", type: "Pipe" },
{ value: "length", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
],
FILTER_OPERATOR_3: [
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "-1", type: "NumericLiteral" },
{ value: "|", type: "Pipe" },
{ value: "abs", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
{ value: "{{", type: "OpenExpression" },
{ value: "1", type: "NumericLiteral" },
{ value: "|", type: "Pipe" },
{ value: "abs", type: "Identifier" },
{ value: "}}", type: "CloseExpression" },
{ value: "|", type: "Text" },
],

// Logical operators between non-Booleans
BOOLEAN_NUMERICAL: [
Expand Down Expand Up @@ -1518,6 +1584,8 @@ const TEST_CONTEXT = {
FILTER_OPERATOR: {
arr: [3, 2, 1],
},
FILTER_OPERATOR_2: {},
FILTER_OPERATOR_3: {},

// Logical operators between non-Booleans
BOOLEAN_NUMERICAL: {},
Expand Down Expand Up @@ -1594,6 +1662,8 @@ const EXPECTED_OUTPUTS = {

// Filter operator
FILTER_OPERATOR: `3451`,
FILTER_OPERATOR_2: `|3|ABCD|abcd|Test test|Test Test|a b|4|`,
FILTER_OPERATOR_3: `|1|1|`,

// Logical operators between non-Booleans
BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`,
Expand Down

0 comments on commit 65b4dad

Please sign in to comment.