From 341f576ec828457a599a60fddd567c090c9e6d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Tavares?= Date: Tue, 11 Jun 2024 10:56:30 +0100 Subject: [PATCH] test262 now compatible with new output format --- test/normalization/test262/run.py | 12 ++- test/results.txt | 160 +++++++++++++++--------------- 2 files changed, 90 insertions(+), 82 deletions(-) diff --git a/test/normalization/test262/run.py b/test/normalization/test262/run.py index e27641e..d8111bd 100644 --- a/test/normalization/test262/run.py +++ b/test/normalization/test262/run.py @@ -1,9 +1,12 @@ import argparse from pathlib import Path +import os import subprocess TIMEOUT = 10 +OUTPUT_PATH = "out/" +NORMALIZED_CODE = f"{OUTPUT_PATH}/graph/normalized.js" # paths TESTS = "tests" @@ -33,9 +36,10 @@ def normalize(path): try: # run normalization - result = subprocess.run(["ast_gen", path], capture_output=True, text=True, check=True, timeout=TIMEOUT) - norm_program = result.stdout - + subprocess.run(["ast_gen", path, "-o", OUTPUT_PATH], capture_output=True, text=True, check=True, timeout=TIMEOUT) + with open(NORMALIZED_CODE, "r") as file: + norm_program = file.read() + return PASS, norm_program except subprocess.CalledProcessError: info = FAIL @@ -172,6 +176,8 @@ def main(): # output detailed report report(info, out_file) + + os.system(f"rm -fr {OUTPUT_PATH}") if __name__ == "__main__": diff --git a/test/results.txt b/test/results.txt index f623b6f..862a3ae 100644 --- a/test/results.txt +++ b/test/results.txt @@ -1,3 +1,4 @@ +[NOK] language/statements/with/S12.10_A3.6_T1.js norm [fail] semantics [] [NOK] language/statements/with/12.10-0-7.js norm [pass] semantics [fail] [eval]:15 throw new Test262Error(message); @@ -7,6 +8,7 @@ Test262Error { message: 'Expected true but got false' } Node.js v21.6.2 +[NOK] language/statements/empty/S12.3_A1.js norm [fail] semantics [] [NOK] language/statements/async-function/syntax-declaration-no-line-terminator.js norm [pass] semantics [fail] [eval]:15 throw new Test262Error(message); @@ -456,7 +458,7 @@ SyntaxError: Unexpected token '}' Node.js v21.6.2 [NOK] language/statements/labeled/value-await-non-module.js norm [pass] semantics [fail] -[eval]:391 +[eval]:390 @@ -471,7 +473,7 @@ SyntaxError: Unexpected end of input Node.js v21.6.2 [NOK] language/statements/labeled/value-await-non-module-escaped.js norm [pass] semantics [fail] -[eval]:391 +[eval]:390 @@ -5113,86 +5115,86 @@ Test262Error { Node.js v21.6.2 -(PASSED 99.3%) language/statements/with failed : 0/1 timeout : 0 -(PASSED 100.0%) language/statements/empty failed : 0/0 timeout : 0 -(PASSED 100.0%) language/statements/variable failed : 0/0 timeout : 0 -(PASSED 0.0%) language/statements/async-function failed : 0/1 timeout : 0 -(PASSED 75.8%) language/statements/if failed : 8/8 timeout : 0 -(PASSED 100.0%) language/statements/return failed : 0/0 timeout : 0 -(PASSED 86.8%) language/statements/function failed : 0/31 timeout : 0 -(PASSED 50.0%) language/statements/labeled failed : 0/3 timeout : 0 -(PASSED 100.0%) language/statements/throw failed : 0/0 timeout : 0 -(PASSED 79.2%) language/statements/do-while failed : 0/5 timeout : 0 -(PASSED 91.0%) language/statements/for failed : 0/4 timeout : 2 -(PASSED 92.9%) language/statements/switch failed : 1/1 timeout : 0 -(PASSED 100.0%) language/statements/switch/syntax/redeclaration failed : 0/0 timeout : 0 -(PASSED 88.1%) language/statements/try failed : 0/2 timeout : 8 -(PASSED 100.0%) language/statements/expression failed : 0/0 timeout : 0 -(PASSED 70.0%) language/statements/break failed : 0/6 timeout : 0 -(PASSED 94.1%) language/statements/for-in failed : 0/2 timeout : 0 -(PASSED 81.2%) language/statements/continue failed : 0/1 timeout : 2 -(PASSED 80.0%) language/statements/while failed : 0/5 timeout : 0 -(PASSED 100.0%) language/statements/block failed : 0/0 timeout : 0 -(PASSED 93.9%) language/expressions/modulus failed : 0/2 timeout : 0 -(PASSED 91.3%) language/expressions/bitwise-and failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/dynamic-import failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/dynamic-import/namespace failed : 0/0 timeout : 0 -(PASSED 0.0%) language/expressions/dynamic-import/catch failed : 0/1 timeout : 0 -(PASSED 100.0%) language/expressions/dynamic-import/syntax/valid failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/relational failed : 0/0 timeout : 0 -(PASSED 94.6%) language/expressions/less-than failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/concatenation failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/unary-plus failed : 0/0 timeout : 0 -(PASSED 68.3%) language/expressions/postfix-increment failed : 0/13 timeout : 0 -(PASSED 100.0%) language/expressions/property-accessors failed : 0/0 timeout : 0 -(PASSED 87.5%) language/expressions/logical-and failed : 0/2 timeout : 0 -(PASSED 94.7%) language/expressions/division failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/conditional failed : 0/0 timeout : 0 -(PASSED 94.9%) language/expressions/instanceof failed : 0/2 timeout : 0 -(PASSED 61.8%) language/expressions/delete failed : 0/34 timeout : 0 -(PASSED 71.8%) language/expressions/postfix-decrement failed : 0/11 timeout : 0 -(PASSED 94.6%) language/expressions/greater-than-or-equal failed : 0/2 timeout : 0 -(PASSED 93.8%) language/expressions/subtraction failed : 0/2 timeout : 0 -(PASSED 45.5%) language/expressions/array failed : 0/6 timeout : 0 -(PASSED 74.2%) language/expressions/prefix-increment failed : 0/8 timeout : 0 -(PASSED 93.3%) language/expressions/does-not-equals failed : 0/2 timeout : 0 -(PASSED 85.7%) language/expressions/in failed : 0/2 timeout : 0 -(PASSED 90.0%) language/expressions/grouping failed : 0/1 timeout : 0 -(PASSED 81.2%) language/expressions/prefix-decrement failed : 0/6 timeout : 0 -(PASSED 79.7%) language/expressions/assignment failed : 1/11 timeout : 0 -(PASSED 100.0%) language/expressions/comma failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/exponentiation failed : 0/0 timeout : 0 -(PASSED 95.2%) language/expressions/function failed : 0/1 timeout : 0 -(PASSED 90.9%) language/expressions/strict-does-not-equals failed : 0/2 timeout : 0 -(PASSED 94.7%) language/expressions/unsigned-right-shift failed : 0/2 timeout : 0 -(PASSED 93.5%) language/expressions/equals failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/logical-not failed : 0/0 timeout : 0 -(PASSED 91.3%) language/expressions/bitwise-xor failed : 0/2 timeout : 0 -(PASSED 76.3%) language/expressions/call failed : 5/4 timeout : 0 -(PASSED 95.1%) language/expressions/greater-than failed : 0/2 timeout : 0 -(PASSED 96.2%) language/expressions/object failed : 0/2 timeout : 0 -(PASSED 91.3%) language/expressions/bitwise-or failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/void failed : 0/0 timeout : 0 -(PASSED 95.2%) language/expressions/less-than-or-equal failed : 0/2 timeout : 0 -(PASSED 90.9%) language/expressions/strict-equals failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/import.meta/syntax failed : 0/0 timeout : 0 -(PASSED 47.1%) language/expressions/new failed : 5/4 timeout : 0 -(PASSED 43.7%) language/expressions/compound-assignment failed : 11/231 timeout : 0 -(PASSED 100.0%) language/expressions/this failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/bitwise-not failed : 0/0 timeout : 0 -(PASSED 87.5%) language/expressions/logical-or failed : 0/2 timeout : 0 -(PASSED 93.3%) language/expressions/right-shift failed : 0/2 timeout : 0 -(PASSED 94.1%) language/expressions/multiplication failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/unary-minus failed : 0/0 timeout : 0 -(PASSED 100.0%) language/expressions/assignmenttargettype failed : 0/0 timeout : 0 -(PASSED 94.7%) language/expressions/left-shift failed : 0/2 timeout : 0 -(PASSED 100.0%) language/expressions/typeof failed : 0/0 timeout : 0 -(PASSED 95.1%) language/expressions/addition failed : 0/2 timeout : 0 +(PASSED 98.7%) language/statements/with failed : 1/1 timeout : 0 +(PASSED 0.0%) language/statements/empty failed : 1/0 timeout : 0 +(PASSED 100.0%) language/statements/variable failed : 0/0 timeout : 0 +(PASSED 0.0%) language/statements/async-function failed : 0/1 timeout : 0 +(PASSED 75.8%) language/statements/if failed : 8/8 timeout : 0 +(PASSED 100.0%) language/statements/return failed : 0/0 timeout : 0 +(PASSED 86.8%) language/statements/function failed : 0/31 timeout : 0 +(PASSED 50.0%) language/statements/labeled failed : 0/3 timeout : 0 +(PASSED 100.0%) language/statements/throw failed : 0/0 timeout : 0 +(PASSED 79.2%) language/statements/do-while failed : 0/5 timeout : 0 +(PASSED 91.0%) language/statements/for failed : 0/4 timeout : 2 +(PASSED 92.9%) language/statements/switch failed : 1/1 timeout : 0 +(PASSED 100.0%) language/statements/switch/syntax/redeclaration failed : 0/0 timeout : 0 +(PASSED 88.1%) language/statements/try failed : 0/2 timeout : 8 +(PASSED 100.0%) language/statements/expression failed : 0/0 timeout : 0 +(PASSED 70.0%) language/statements/break failed : 0/6 timeout : 0 +(PASSED 94.1%) language/statements/for-in failed : 0/2 timeout : 0 +(PASSED 81.2%) language/statements/continue failed : 0/1 timeout : 2 +(PASSED 80.0%) language/statements/while failed : 0/5 timeout : 0 +(PASSED 100.0%) language/statements/block failed : 0/0 timeout : 0 +(PASSED 93.9%) language/expressions/modulus failed : 0/2 timeout : 0 +(PASSED 91.3%) language/expressions/bitwise-and failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/dynamic-import failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/dynamic-import/namespace failed : 0/0 timeout : 0 +(PASSED 0.0%) language/expressions/dynamic-import/catch failed : 0/1 timeout : 0 +(PASSED 100.0%) language/expressions/dynamic-import/syntax/valid failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/relational failed : 0/0 timeout : 0 +(PASSED 94.6%) language/expressions/less-than failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/concatenation failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/unary-plus failed : 0/0 timeout : 0 +(PASSED 68.3%) language/expressions/postfix-increment failed : 0/13 timeout : 0 +(PASSED 100.0%) language/expressions/property-accessors failed : 0/0 timeout : 0 +(PASSED 87.5%) language/expressions/logical-and failed : 0/2 timeout : 0 +(PASSED 94.7%) language/expressions/division failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/conditional failed : 0/0 timeout : 0 +(PASSED 94.9%) language/expressions/instanceof failed : 0/2 timeout : 0 +(PASSED 61.8%) language/expressions/delete failed : 0/34 timeout : 0 +(PASSED 71.8%) language/expressions/postfix-decrement failed : 0/11 timeout : 0 +(PASSED 94.6%) language/expressions/greater-than-or-equal failed : 0/2 timeout : 0 +(PASSED 93.8%) language/expressions/subtraction failed : 0/2 timeout : 0 +(PASSED 45.5%) language/expressions/array failed : 0/6 timeout : 0 +(PASSED 74.2%) language/expressions/prefix-increment failed : 0/8 timeout : 0 +(PASSED 93.3%) language/expressions/does-not-equals failed : 0/2 timeout : 0 +(PASSED 85.7%) language/expressions/in failed : 0/2 timeout : 0 +(PASSED 90.0%) language/expressions/grouping failed : 0/1 timeout : 0 +(PASSED 81.2%) language/expressions/prefix-decrement failed : 0/6 timeout : 0 +(PASSED 79.7%) language/expressions/assignment failed : 1/11 timeout : 0 +(PASSED 100.0%) language/expressions/comma failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/exponentiation failed : 0/0 timeout : 0 +(PASSED 95.2%) language/expressions/function failed : 0/1 timeout : 0 +(PASSED 90.9%) language/expressions/strict-does-not-equals failed : 0/2 timeout : 0 +(PASSED 94.7%) language/expressions/unsigned-right-shift failed : 0/2 timeout : 0 +(PASSED 93.5%) language/expressions/equals failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/logical-not failed : 0/0 timeout : 0 +(PASSED 91.3%) language/expressions/bitwise-xor failed : 0/2 timeout : 0 +(PASSED 76.3%) language/expressions/call failed : 5/4 timeout : 0 +(PASSED 95.1%) language/expressions/greater-than failed : 0/2 timeout : 0 +(PASSED 96.2%) language/expressions/object failed : 0/2 timeout : 0 +(PASSED 91.3%) language/expressions/bitwise-or failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/void failed : 0/0 timeout : 0 +(PASSED 95.2%) language/expressions/less-than-or-equal failed : 0/2 timeout : 0 +(PASSED 90.9%) language/expressions/strict-equals failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/import.meta/syntax failed : 0/0 timeout : 0 +(PASSED 47.1%) language/expressions/new failed : 5/4 timeout : 0 +(PASSED 43.7%) language/expressions/compound-assignment failed : 11/231 timeout : 0 +(PASSED 100.0%) language/expressions/this failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/bitwise-not failed : 0/0 timeout : 0 +(PASSED 87.5%) language/expressions/logical-or failed : 0/2 timeout : 0 +(PASSED 93.3%) language/expressions/right-shift failed : 0/2 timeout : 0 +(PASSED 94.1%) language/expressions/multiplication failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/unary-minus failed : 0/0 timeout : 0 +(PASSED 100.0%) language/expressions/assignmenttargettype failed : 0/0 timeout : 0 +(PASSED 94.7%) language/expressions/left-shift failed : 0/2 timeout : 0 +(PASSED 100.0%) language/expressions/typeof failed : 0/0 timeout : 0 +(PASSED 95.1%) language/expressions/addition failed : 0/2 timeout : 0 ============================== total : 2561 - ok : 2078 - error : 31 + 449 (normalization + semantics) + ok : 2076 + error : 33 + 449 (normalization + semantics) timeout : 12 ==============================