This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch jenkins-add-test262-to-quickjs in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 826a67eb852823ed37bb4a2906badd56c2a249cf Author: Nick Vatamaniuc <[email protected]> AuthorDate: Fri Jun 20 18:03:17 2025 -0400 Run test262 JS conformance tests for QuickJS Test262 are 70k+ Javascript conformance tests [1]. There is an upstream PR [2] to run these in CI there but until then we can patch our own QuickJS and run them in our CI. One of the patches we have `04-test262-errors.patch` patches one of the exception we expect to get since modified the parser to accept the `function` expression syntax. For now run the test in the full CI only for Linux and skip s390x. Some architectures/os combinations show various discrepancies around TypedArrays or rounding errors. [1] https://github.com/tc39/test262 [2] https://github.com/bellard/quickjs/pull/408 --- Makefile | 6 +++++ build-aux/Jenkinsfile.full | 18 ++++++++++++++ src/couch_quickjs/.gitignore | 2 ++ .../patches/02-test262-makefile.patch | 29 ++++++++++++++++++++++ src/couch_quickjs/patches/03-test262-yield.patch | 15 +++++++++++ src/couch_quickjs/patches/04-test262-errors.patch | 11 ++++++++ src/couch_quickjs/quickjs/Makefile | 13 ++++++++++ src/couch_quickjs/quickjs/test262_errors.txt | 2 ++ src/couch_quickjs/quickjs/tests/test262.patch | 6 ++--- src/couch_quickjs/update_patches.sh | 3 +++ 10 files changed, 102 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index fdab740a0..918234f25 100644 --- a/Makefile +++ b/Makefile @@ -354,6 +354,12 @@ weatherreport-test: devclean escriptize @dev/run "$(TEST_OPTS)" -n 1 -a adm:pass --no-eval \ 'bin/weatherreport --etc dev/lib/node1/etc --level error' +.PHONY: quickjs-test262 +# target: quickjs-javascript-tests - Run QuickJS JS conformance tests +quickjs-test262: couch + make -C src/couch_quickjs/quickjs test2-bootstrap + make -C src/couch_quickjs/quickjs test2 + ################################################################################ # Developing ################################################################################ diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full index b955a3d9b..adfdc597c 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile.full @@ -28,6 +28,7 @@ meta = [ spidermonkey_vsn: '60', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" ], @@ -36,6 +37,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" ], @@ -44,6 +46,7 @@ meta = [ spidermonkey_vsn: '91', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" ], @@ -52,6 +55,7 @@ meta = [ spidermonkey_vsn: '115', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" ], @@ -60,6 +64,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", node_label: 'ppc64le' ], @@ -68,6 +73,9 @@ meta = [ name: 'Debian s390x', spidermonkey_vsn: '78', with_nouveau: true, + // QuickJS test262 shows a discrepancy typedarray-arg-set-values-same-buffer-other-type.js + // Test262Error: 51539607552,42,0,4,5,6,7,8 + quickjs_test262: false, image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", node_label: 's390x' ], @@ -77,6 +85,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" ], @@ -85,6 +94,8 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + // Test this in in the bookworm-quickjs variant + quickjs_test262: false, image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" ], @@ -93,6 +104,7 @@ meta = [ disable_spidermonkey: true, with_nouveau: true, with_clouseau: true, + quickjs_test262: true, image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" ], @@ -101,6 +113,7 @@ meta = [ spidermonkey_vsn: '91', with_clouseau: true, clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, gnu_make: 'gmake' ], @@ -111,6 +124,7 @@ meta = [ disable_spidermonkey: true, with_clouseau: true, clouseau_java_home: '/usr/local/openjdk8-jre', + quickjs_test262: false, gnu_make: 'gmake' ], @@ -130,6 +144,7 @@ meta = [ name: 'Windows 2022', spidermonkey_vsn: '128', with_clouseau: true, + quickjs_test262: false, node_label: 'win' ] ] @@ -187,6 +202,7 @@ def generateNativeStage(platform) { sh "${configure(meta[platform])}" sh '$MAKE' retry (3) {sh '$MAKE eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry (3) {sh '$MAKE elixir'} retry (3) {sh '$MAKE elixir-search'} retry (3) {sh '$MAKE mango-test'} @@ -304,10 +320,12 @@ def generateContainerStage(platform) { unstash 'tarball' sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) + quickjs_tests262 = meta[platform].quickjs_test262 dir( "${platform}/build" ) { sh "${configure(meta[platform])}" sh 'make' retry(3) {sh 'make eunit'} + if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry(3) {sh 'make elixir'} retry(3) {sh 'make elixir-search'} retry(3) {sh 'make mango-test'} diff --git a/src/couch_quickjs/.gitignore b/src/couch_quickjs/.gitignore index dd78afee1..f7d339351 100644 --- a/src/couch_quickjs/.gitignore +++ b/src/couch_quickjs/.gitignore @@ -16,6 +16,8 @@ /quickjs/qjscalc.c /quickjs/repl.c /quickjs/run-test262 +/quickjs/test262_report.txt +/quickjs/test262/ /quickjs/test_fib.c /quickjs/.github compile_commands.json diff --git a/src/couch_quickjs/patches/02-test262-makefile.patch b/src/couch_quickjs/patches/02-test262-makefile.patch new file mode 100644 index 000000000..1bfc17640 --- /dev/null +++ b/src/couch_quickjs/patches/02-test262-makefile.patch @@ -0,0 +1,29 @@ +--- quickjs-master/Makefile 2025-06-14 05:51:48 ++++ quickjs/Makefile 2025-06-20 18:03:41 +@@ -53,6 +53,10 @@ + #CONFIG_MSAN=y + # use UB sanitizer + #CONFIG_UBSAN=y ++ ++# TEST262 bootstrap config: commit id and shallow "since" parameter ++TEST262_COMMIT?=3316c0aaf676d657f5a6b33364fa7e579c78ac7f ++TEST262_SINCE?=2025-05-21 + + OBJDIR=.obj + +@@ -464,6 +468,15 @@ + microbench: qjs$(EXE) + $(WINE) ./qjs$(EXE) --std tests/microbench.js + ++ifeq ($(wildcard test262/features.txt),) ++test2-bootstrap: ++ git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git ++ (cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) ++else ++test2-bootstrap: ++ (cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) ++endif ++ + ifeq ($(wildcard test262o/tests.txt),) + test2o test2o-update: + @echo test262o tests not installed diff --git a/src/couch_quickjs/patches/03-test262-yield.patch b/src/couch_quickjs/patches/03-test262-yield.patch new file mode 100644 index 000000000..e4dc4c5be --- /dev/null +++ b/src/couch_quickjs/patches/03-test262-yield.patch @@ -0,0 +1,15 @@ +--- quickjs-master/tests/test262.patch 2025-06-14 05:51:48 ++++ quickjs/tests/test262.patch 2025-06-20 18:03:41 +@@ -14,9 +14,9 @@ + +// small: 200, + +// long: 1000, + +// huge: 10000, +-+ yield: 20, +-+ small: 20, +-+ long: 100, +++ yield: 40, +++ small: 40, +++ long: 200, + + huge: 1000, + }; + diff --git a/src/couch_quickjs/patches/04-test262-errors.patch b/src/couch_quickjs/patches/04-test262-errors.patch new file mode 100644 index 000000000..3de2591e6 --- /dev/null +++ b/src/couch_quickjs/patches/04-test262-errors.patch @@ -0,0 +1,11 @@ +--- quickjs-master/test262_errors.txt 2025-06-14 05:51:48 ++++ quickjs/test262_errors.txt 2025-06-20 18:03:41 +@@ -1,6 +1,8 @@ + test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached + test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached + test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called ++test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated. ++test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated. + test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true + test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true + test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true diff --git a/src/couch_quickjs/quickjs/Makefile b/src/couch_quickjs/quickjs/Makefile index 3b1c74594..fd74c7be2 100644 --- a/src/couch_quickjs/quickjs/Makefile +++ b/src/couch_quickjs/quickjs/Makefile @@ -54,6 +54,10 @@ PREFIX?=/usr/local # use UB sanitizer #CONFIG_UBSAN=y +# TEST262 bootstrap config: commit id and shallow "since" parameter +TEST262_COMMIT?=3316c0aaf676d657f5a6b33364fa7e579c78ac7f +TEST262_SINCE?=2025-05-21 + OBJDIR=.obj ifdef CONFIG_ASAN @@ -464,6 +468,15 @@ stats: qjs$(EXE) microbench: qjs$(EXE) $(WINE) ./qjs$(EXE) --std tests/microbench.js +ifeq ($(wildcard test262/features.txt),) +test2-bootstrap: + git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git + (cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) +else +test2-bootstrap: + (cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..) +endif + ifeq ($(wildcard test262o/tests.txt),) test2o test2o-update: @echo test262o tests not installed diff --git a/src/couch_quickjs/quickjs/test262_errors.txt b/src/couch_quickjs/quickjs/test262_errors.txt index 7dd3f7624..a252d6ec3 100644 --- a/src/couch_quickjs/quickjs/test262_errors.txt +++ b/src/couch_quickjs/quickjs/test262_errors.txt @@ -1,6 +1,8 @@ test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called +test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated. +test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated. test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true diff --git a/src/couch_quickjs/quickjs/tests/test262.patch b/src/couch_quickjs/quickjs/tests/test262.patch index 4ed0afb8e..c8885e658 100644 --- a/src/couch_quickjs/quickjs/tests/test262.patch +++ b/src/couch_quickjs/quickjs/tests/test262.patch @@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644 +// small: 200, +// long: 1000, +// huge: 10000, -+ yield: 20, -+ small: 20, -+ long: 100, ++ yield: 40, ++ small: 40, ++ long: 200, + huge: 1000, }; diff --git a/src/couch_quickjs/update_patches.sh b/src/couch_quickjs/update_patches.sh index f14b5ab6f..396811a85 100755 --- a/src/couch_quickjs/update_patches.sh +++ b/src/couch_quickjs/update_patches.sh @@ -17,6 +17,9 @@ unzip -q -o master.zip echo " * updating 01-spidermonkey-185-mode.patch" set +e diff -u quickjs-master/quickjs.c quickjs/quickjs.c > patches/01-spidermonkey-185-mode.patch +diff -u quickjs-master/Makefile quickjs/Makefile > patches/02-test262-makefile.patch +diff -u quickjs-master/tests/test262.patch quickjs/tests/test262.patch > patches/03-test262-yield.patch +diff -u quickjs-master/test262_errors.txt quickjs/test262_errors.txt > patches/04-test262-errors.patch set -e echo " * cleaning up" rm -rf master.zip quickjs-master
