Skip to content

Commit

Permalink
test262 now compatible with new output format
Browse files Browse the repository at this point in the history
  • Loading branch information
@Th0mz
Th0mz committed Jun 11, 2024
1 parent b4e9f46 commit 341f576
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 82 deletions.
12 changes: 9 additions & 3 deletions test/normalization/test262/run.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import argparse
from pathlib import Path
import os
import subprocess


TIMEOUT = 10
OUTPUT_PATH = "out/"
NORMALIZED_CODE = f"{OUTPUT_PATH}/graph/normalized.js"

# paths
TESTS = "tests"
Expand Down Expand Up @@ -33,9 +36,10 @@
def normalize(path):
try:
# run normalization
result = subprocess.run(["ast_gen", path], capture_output=True, text=True, check=True, timeout=TIMEOUT)
norm_program = result.stdout

subprocess.run(["ast_gen", path, "-o", OUTPUT_PATH], capture_output=True, text=True, check=True, timeout=TIMEOUT)
with open(NORMALIZED_CODE, "r") as file:
norm_program = file.read()

return PASS, norm_program
except subprocess.CalledProcessError:
info = FAIL
Expand Down Expand Up @@ -172,6 +176,8 @@ def main():

# output detailed report
report(info, out_file)

os.system(f"rm -fr {OUTPUT_PATH}")


if __name__ == "__main__":
Expand Down
160 changes: 81 additions & 79 deletions test/results.txt
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[NOK] language/statements/with/S12.10_A3.6_T1.js norm [fail] semantics []
[NOK] language/statements/with/12.10-0-7.js norm [pass] semantics [fail]
[eval]:15
throw new Test262Error(message);
Expand All @@ -7,6 +8,7 @@ Test262Error { message: 'Expected true but got false' }

Node.js v21.6.2

[NOK] language/statements/empty/S12.3_A1.js norm [fail] semantics []
[NOK] language/statements/async-function/syntax-declaration-no-line-terminator.js norm [pass] semantics [fail]
[eval]:15
throw new Test262Error(message);
Expand Down Expand Up @@ -456,7 +458,7 @@ SyntaxError: Unexpected token '}'
Node.js v21.6.2

[NOK] language/statements/labeled/value-await-non-module.js norm [pass] semantics [fail]
[eval]:391
[eval]:390



Expand All @@ -471,7 +473,7 @@ SyntaxError: Unexpected end of input
Node.js v21.6.2

[NOK] language/statements/labeled/value-await-non-module-escaped.js norm [pass] semantics [fail]
[eval]:391
[eval]:390



Expand Down Expand Up @@ -5113,86 +5115,86 @@ Test262Error {

Node.js v21.6.2

(PASSED 99.3%) language/statements/with failed : 0/1 timeout : 0
(PASSED 100.0%) language/statements/empty failed : 0/0 timeout : 0
(PASSED 100.0%) language/statements/variable failed : 0/0 timeout : 0
(PASSED 0.0%) language/statements/async-function failed : 0/1 timeout : 0
(PASSED 75.8%) language/statements/if failed : 8/8 timeout : 0
(PASSED 100.0%) language/statements/return failed : 0/0 timeout : 0
(PASSED 86.8%) language/statements/function failed : 0/31 timeout : 0
(PASSED 50.0%) language/statements/labeled failed : 0/3 timeout : 0
(PASSED 100.0%) language/statements/throw failed : 0/0 timeout : 0
(PASSED 79.2%) language/statements/do-while failed : 0/5 timeout : 0
(PASSED 91.0%) language/statements/for failed : 0/4 timeout : 2
(PASSED 92.9%) language/statements/switch failed : 1/1 timeout : 0
(PASSED 100.0%) language/statements/switch/syntax/redeclaration failed : 0/0 timeout : 0
(PASSED 88.1%) language/statements/try failed : 0/2 timeout : 8
(PASSED 100.0%) language/statements/expression failed : 0/0 timeout : 0
(PASSED 70.0%) language/statements/break failed : 0/6 timeout : 0
(PASSED 94.1%) language/statements/for-in failed : 0/2 timeout : 0
(PASSED 81.2%) language/statements/continue failed : 0/1 timeout : 2
(PASSED 80.0%) language/statements/while failed : 0/5 timeout : 0
(PASSED 100.0%) language/statements/block failed : 0/0 timeout : 0
(PASSED 93.9%) language/expressions/modulus failed : 0/2 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-and failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import/namespace failed : 0/0 timeout : 0
(PASSED 0.0%) language/expressions/dynamic-import/catch failed : 0/1 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import/syntax/valid failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/relational failed : 0/0 timeout : 0
(PASSED 94.6%) language/expressions/less-than failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/concatenation failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/unary-plus failed : 0/0 timeout : 0
(PASSED 68.3%) language/expressions/postfix-increment failed : 0/13 timeout : 0
(PASSED 100.0%) language/expressions/property-accessors failed : 0/0 timeout : 0
(PASSED 87.5%) language/expressions/logical-and failed : 0/2 timeout : 0
(PASSED 94.7%) language/expressions/division failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/conditional failed : 0/0 timeout : 0
(PASSED 94.9%) language/expressions/instanceof failed : 0/2 timeout : 0
(PASSED 61.8%) language/expressions/delete failed : 0/34 timeout : 0
(PASSED 71.8%) language/expressions/postfix-decrement failed : 0/11 timeout : 0
(PASSED 94.6%) language/expressions/greater-than-or-equal failed : 0/2 timeout : 0
(PASSED 93.8%) language/expressions/subtraction failed : 0/2 timeout : 0
(PASSED 45.5%) language/expressions/array failed : 0/6 timeout : 0
(PASSED 74.2%) language/expressions/prefix-increment failed : 0/8 timeout : 0
(PASSED 93.3%) language/expressions/does-not-equals failed : 0/2 timeout : 0
(PASSED 85.7%) language/expressions/in failed : 0/2 timeout : 0
(PASSED 90.0%) language/expressions/grouping failed : 0/1 timeout : 0
(PASSED 81.2%) language/expressions/prefix-decrement failed : 0/6 timeout : 0
(PASSED 79.7%) language/expressions/assignment failed : 1/11 timeout : 0
(PASSED 100.0%) language/expressions/comma failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/exponentiation failed : 0/0 timeout : 0
(PASSED 95.2%) language/expressions/function failed : 0/1 timeout : 0
(PASSED 90.9%) language/expressions/strict-does-not-equals failed : 0/2 timeout : 0
(PASSED 94.7%) language/expressions/unsigned-right-shift failed : 0/2 timeout : 0
(PASSED 93.5%) language/expressions/equals failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/logical-not failed : 0/0 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-xor failed : 0/2 timeout : 0
(PASSED 76.3%) language/expressions/call failed : 5/4 timeout : 0
(PASSED 95.1%) language/expressions/greater-than failed : 0/2 timeout : 0
(PASSED 96.2%) language/expressions/object failed : 0/2 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-or failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/void failed : 0/0 timeout : 0
(PASSED 95.2%) language/expressions/less-than-or-equal failed : 0/2 timeout : 0
(PASSED 90.9%) language/expressions/strict-equals failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/import.meta/syntax failed : 0/0 timeout : 0
(PASSED 47.1%) language/expressions/new failed : 5/4 timeout : 0
(PASSED 43.7%) language/expressions/compound-assignment failed : 11/231 timeout : 0
(PASSED 100.0%) language/expressions/this failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/bitwise-not failed : 0/0 timeout : 0
(PASSED 87.5%) language/expressions/logical-or failed : 0/2 timeout : 0
(PASSED 93.3%) language/expressions/right-shift failed : 0/2 timeout : 0
(PASSED 94.1%) language/expressions/multiplication failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/unary-minus failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/assignmenttargettype failed : 0/0 timeout : 0
(PASSED 94.7%) language/expressions/left-shift failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/typeof failed : 0/0 timeout : 0
(PASSED 95.1%) language/expressions/addition failed : 0/2 timeout : 0
(PASSED 98.7%) language/statements/with failed : 1/1 timeout : 0
(PASSED 0.0%) language/statements/empty failed : 1/0 timeout : 0
(PASSED 100.0%) language/statements/variable failed : 0/0 timeout : 0
(PASSED 0.0%) language/statements/async-function failed : 0/1 timeout : 0
(PASSED 75.8%) language/statements/if failed : 8/8 timeout : 0
(PASSED 100.0%) language/statements/return failed : 0/0 timeout : 0
(PASSED 86.8%) language/statements/function failed : 0/31 timeout : 0
(PASSED 50.0%) language/statements/labeled failed : 0/3 timeout : 0
(PASSED 100.0%) language/statements/throw failed : 0/0 timeout : 0
(PASSED 79.2%) language/statements/do-while failed : 0/5 timeout : 0
(PASSED 91.0%) language/statements/for failed : 0/4 timeout : 2
(PASSED 92.9%) language/statements/switch failed : 1/1 timeout : 0
(PASSED 100.0%) language/statements/switch/syntax/redeclaration failed : 0/0 timeout : 0
(PASSED 88.1%) language/statements/try failed : 0/2 timeout : 8
(PASSED 100.0%) language/statements/expression failed : 0/0 timeout : 0
(PASSED 70.0%) language/statements/break failed : 0/6 timeout : 0
(PASSED 94.1%) language/statements/for-in failed : 0/2 timeout : 0
(PASSED 81.2%) language/statements/continue failed : 0/1 timeout : 2
(PASSED 80.0%) language/statements/while failed : 0/5 timeout : 0
(PASSED 100.0%) language/statements/block failed : 0/0 timeout : 0
(PASSED 93.9%) language/expressions/modulus failed : 0/2 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-and failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import/namespace failed : 0/0 timeout : 0
(PASSED 0.0%) language/expressions/dynamic-import/catch failed : 0/1 timeout : 0
(PASSED 100.0%) language/expressions/dynamic-import/syntax/valid failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/relational failed : 0/0 timeout : 0
(PASSED 94.6%) language/expressions/less-than failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/concatenation failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/unary-plus failed : 0/0 timeout : 0
(PASSED 68.3%) language/expressions/postfix-increment failed : 0/13 timeout : 0
(PASSED 100.0%) language/expressions/property-accessors failed : 0/0 timeout : 0
(PASSED 87.5%) language/expressions/logical-and failed : 0/2 timeout : 0
(PASSED 94.7%) language/expressions/division failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/conditional failed : 0/0 timeout : 0
(PASSED 94.9%) language/expressions/instanceof failed : 0/2 timeout : 0
(PASSED 61.8%) language/expressions/delete failed : 0/34 timeout : 0
(PASSED 71.8%) language/expressions/postfix-decrement failed : 0/11 timeout : 0
(PASSED 94.6%) language/expressions/greater-than-or-equal failed : 0/2 timeout : 0
(PASSED 93.8%) language/expressions/subtraction failed : 0/2 timeout : 0
(PASSED 45.5%) language/expressions/array failed : 0/6 timeout : 0
(PASSED 74.2%) language/expressions/prefix-increment failed : 0/8 timeout : 0
(PASSED 93.3%) language/expressions/does-not-equals failed : 0/2 timeout : 0
(PASSED 85.7%) language/expressions/in failed : 0/2 timeout : 0
(PASSED 90.0%) language/expressions/grouping failed : 0/1 timeout : 0
(PASSED 81.2%) language/expressions/prefix-decrement failed : 0/6 timeout : 0
(PASSED 79.7%) language/expressions/assignment failed : 1/11 timeout : 0
(PASSED 100.0%) language/expressions/comma failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/exponentiation failed : 0/0 timeout : 0
(PASSED 95.2%) language/expressions/function failed : 0/1 timeout : 0
(PASSED 90.9%) language/expressions/strict-does-not-equals failed : 0/2 timeout : 0
(PASSED 94.7%) language/expressions/unsigned-right-shift failed : 0/2 timeout : 0
(PASSED 93.5%) language/expressions/equals failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/logical-not failed : 0/0 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-xor failed : 0/2 timeout : 0
(PASSED 76.3%) language/expressions/call failed : 5/4 timeout : 0
(PASSED 95.1%) language/expressions/greater-than failed : 0/2 timeout : 0
(PASSED 96.2%) language/expressions/object failed : 0/2 timeout : 0
(PASSED 91.3%) language/expressions/bitwise-or failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/void failed : 0/0 timeout : 0
(PASSED 95.2%) language/expressions/less-than-or-equal failed : 0/2 timeout : 0
(PASSED 90.9%) language/expressions/strict-equals failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/import.meta/syntax failed : 0/0 timeout : 0
(PASSED 47.1%) language/expressions/new failed : 5/4 timeout : 0
(PASSED 43.7%) language/expressions/compound-assignment failed : 11/231 timeout : 0
(PASSED 100.0%) language/expressions/this failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/bitwise-not failed : 0/0 timeout : 0
(PASSED 87.5%) language/expressions/logical-or failed : 0/2 timeout : 0
(PASSED 93.3%) language/expressions/right-shift failed : 0/2 timeout : 0
(PASSED 94.1%) language/expressions/multiplication failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/unary-minus failed : 0/0 timeout : 0
(PASSED 100.0%) language/expressions/assignmenttargettype failed : 0/0 timeout : 0
(PASSED 94.7%) language/expressions/left-shift failed : 0/2 timeout : 0
(PASSED 100.0%) language/expressions/typeof failed : 0/0 timeout : 0
(PASSED 95.1%) language/expressions/addition failed : 0/2 timeout : 0

==============================
total : 2561
ok : 2078
error : 31 + 449 (normalization + semantics)
ok : 2076
error : 33 + 449 (normalization + semantics)
timeout : 12
==============================

0 comments on commit 341f576

Please sign in to comment.