More robust & complete conversion of WPT tests.

Now convert.js knows how to deal with subdirectories and .all.js
tests.

Also, instead of carrying our own hacked-up copies of support.js and
support-promises.js, now support scripts are bundled into the test
directly, and the only import is to set up a WPT-like environment.

package.json

@@ -40,6 +40,7 @@
     ],
     "dependencies": {
         "core-js": "^2.4.1",
+        "glob": "^7.1.3",
         "realistic-structured-clone": "^2.0.1",
         "setimmediate": "^1.0.5"
     },

src/test/web-platform-tests/README.md

@@ -1,3 +1,3 @@
-These tests come from [web-platform-tests](https://github.com/w3c/web-platform-tests/tree/master/IndexedDB) as of April 2017.
+These tests come from [web-platform-tests](https://github.com/w3c/web-platform-tests/tree/master/IndexedDB), last copied in March 2019 from commit [`5df243654fcf1672d8e76d461d989651d907873a`](https://github.com/web-platform-tests/wpt/commit/5df243654fcf1672d8e76d461d989651d907873a).
 
-To update them, clone that repo, copy over the IndexedDB folder, and run the convert.js script. Assuming nothing substantial has changed in the structure of the tests, that should be all you have to do.
+To update them, clone that repo, copy over the IndexedDB folder, remove `converted/`, and run the convert.js script. Assuming nothing substantial has changed in the structure of the tests, that should be all you have to do.

src/test/web-platform-tests/convert.js

@@ -1,35 +1,137 @@
 const fs = require("fs");
+const glob = require("glob");
 const path = require("path");
 
-const inFolder = path.join(__dirname, "IndexedDB");
-const outFolder = path.join(__dirname, "converted");
+const skip = [
+    // IDL test; out of scope for the time being.
+    "idlharness.any.js",
+];
 
-const filenames = fs.readdirSync(inFolder);
-for (const filename of filenames) {
-    if (path.extname(filename).indexOf(".htm") !== 0) {
-        continue;
+// It's 2019 and JavaScript is only just implementing a `matchAll`
+// function.
+function* matchAll(string, inputRe) {
+    const re = new RegExp(inputRe, inputRe.flags + "g");
+    while (true) {
+        const match = re.exec(string);
+        if (match === null) {
+            break;
+        }
+        yield match;
     }
-    console.log(`Converting ${filename}...`);
-
-    const contents = fs.readFileSync(path.join(inFolder, filename), "utf8");
-    let matches = contents.match(/<script>([\s\S]+?)<\/script>/); // http://stackoverflow.com/q/1979884/786644
-    if (matches === null || matches.length < 2) {
-        matches = contents.match(
-            /<script type="text\/javascript">([\s\S]+?)<\/script>/,
-        ); // http://stackoverflow.com/q/1979884/786644
+};
+
+function makeParentDir(file) {
+    const dir = path.posix.dirname(file);
+    if (!fs.existsSync(dir)) {
+        makeParentDir(dir);
+        fs.mkdirSync(dir);
     }
-    if (matches === null || matches.length < 2) {
-        throw new Error("No script found");
+}
+
+const inFolder = path.posix.join(__dirname, "IndexedDB");
+const outFolder = path.posix.join(__dirname, "converted");
+
+{
+    const filenames = glob.sync("/**/*.{htm,html}", { root: inFolder });
+    for (const filename of filenames) {
+        const relative = path.posix.relative(inFolder, filename);
+        if (skip.includes(relative)) {
+            console.log(`Skipping ${relative}.`);
+            continue;
+        }
+        const { dir, name } = path.parse(relative);
+        const dest = path.join(outFolder, dir, `${name}.js`);
+        console.log(`Converting ${relative}...`);
+
+        const contents = fs.readFileSync(filename, "utf8");
+        let matches = contents.match(/<script>([\s\S]+?)<\/script>/); // http://stackoverflow.com/q/1979884/786644
+        if (matches === null || matches.length < 2) {
+            matches = contents.match(
+                    /<script type="text\/javascript">([\s\S]+?)<\/script>/,
+            ); // http://stackoverflow.com/q/1979884/786644
+        }
+        if (matches === null || matches.length < 2) {
+            throw new Error("No script found");
+        }
+
+        const testScript = matches[1];
+
+        const codeChunks = [];
+
+        {
+            const relativeWptEnvLocation = path.posix.join(
+                path.posix.relative(path.posix.dirname(dest), __dirname),
+                "wpt-env.js",
+            );
+            codeChunks.push(`require("${relativeWptEnvLocation}");\n`);
+        }
+
+        // Because these are 'imported' with <script>, the support
+        // scripts share a scope with the test script, and that's how
+        // the utilities are accessed. The simplest way to emulate the
+        // browser behaviour here is to glom it all into a single
+        // file.
+
+        const importMatches = matchAll(contents, /<script src=["']?(.+?)['"]?>/);
+
+        for (const match of importMatches) {
+            if (match[1] === "/resources/testharness.js") {
+                continue;
+            }
+            if (match[1] === "/resources/testharnessreport.js") {
+                continue;
+            }
+            const location = path.posix.join(path.posix.dirname(filename), match[1]);
+            codeChunks.push(fs.readFileSync(location) + "\n");
+        }
+
+        codeChunks.push(testScript);
+
+        makeParentDir(dest);
+
+        fs.writeFileSync(dest, codeChunks.join("\n"));
     }
+}
+
+{
+    const filenames = glob.sync("/**/*.any.js", { root: inFolder });
+    for (const filename of filenames) {
+        const relative = path.posix.relative(inFolder, filename);
+        if (skip.includes(relative)) {
+            console.log(`Skipping ${relative}.`);
+            continue;
+        }
+        const { dir, name } = path.parse(relative);
+        const dest = path.join(outFolder, dir, `${name}.js`);
+
+        console.log(`Converting ${relative}...`);
+
+        const testScript = fs.readFileSync(filename, "utf8");
 
-    const testScript = matches[1];
+        // TODO: what does, e.g., 'META: global=window,worker' do? Do
+        // we have to care about it?
 
-    const output = `require("../support-node");
-${testScript}`;
+        const codeChunks = [];
 
-    const baseFilename = path.basename(
-        path.basename(filename, ".htm"),
-        ".html",
-    );
-    fs.writeFileSync(path.join(outFolder, `${baseFilename}.js`), output);
+        {
+            const relativeWptEnvLocation = path.posix.join(
+                path.posix.relative(path.posix.dirname(dest), __dirname),
+                "wpt-env.js",
+            );
+            codeChunks.push(`require("${relativeWptEnvLocation}");\n`);
+        }
+
+        const importMatches = matchAll(testScript, /^\/\/\s*META:\s*script=(.+)$/m);
+
+        for (const match of importMatches) {
+            const location = path.posix.join(path.posix.dirname(filename), match[1]);
+            codeChunks.push(fs.readFileSync(location) + "\n");
+        }
+
+        codeChunks.push(testScript);
+
+        makeParentDir(dest);
+
+        fs.writeFileSync(dest, codeChunks.join("\n"));
+    }
 }

src/test/web-platform-tests/run-all.js

@@ -2,6 +2,7 @@
 
 const fs = require("fs");
 const execSync = require("child_process").execSync;
+const glob = require("glob");
 const path = require("path");
 const semver = require("semver");
 
@@ -105,8 +106,9 @@ const skip = [
     "transaction-abort-request-error.js",
 ];
 
-const filenames = fs.readdirSync(testFolder);
-for (const filename of filenames) {
+const filenames = glob.sync("/**/*.js", { root: testFolder });
+for (const absFilename of filenames) {
+    const filename = path.relative(testFolder, absFilename);
     if (skip.includes(filename)) {
         console.log(`Skipping ${filename}...\n`);
         skipped += 1;