diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4e4d985d..899bf73d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,3 +57,7 @@ jobs: run: | echo "Running in $BROWSER" npm run test:${{ matrix.browser }} + - name: Run end2end + run: | + echo "Running in $BROWSER" + npm run test-e2e:${{ matrix.browser }} diff --git a/package-lock.json b/package-lock.json index 0e5b02985..cdd1e17ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "http-server": "^14.1.1", "mocha": "^10.2.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0", + "selenium-webdriver": "^4.27.0", "sinon": "^17.0.1", "typescript": "^5.0.4" }, @@ -722,6 +722,12 @@ "node": ">=6.9.0" } }, + "node_modules/@bazel/runfiles": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.3.1.tgz", + "integrity": "sha512-1uLNT5NZsUVIGS4syuHwTzZ8HycMPyr6POA3FCE4GbMtc4rhoJk8aZKtNIRthJYfL+iioppi+rTfH3olMPr9nA==", + "dev": true + }, "node_modules/@ember-data/rfc395-data": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/@ember-data/rfc395-data/-/rfc395-data-0.0.4.tgz", @@ -5538,17 +5544,28 @@ "dev": true }, "node_modules/selenium-webdriver": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.0.tgz", - "integrity": "sha512-s/HL8WNwy1ggHR244+tAhjhyKMJnZLt1HKJ6Gn7nQgVjB/ybDF+46Uui0qI2J7AjPNJzlUmTncdC/jg/kKkn0A==", + "version": "4.27.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", + "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/SeleniumHQ" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/selenium" + } + ], "dependencies": { - "jszip": "^3.10.0", - "tmp": "^0.2.1", - "ws": ">=8.11.0" + "@bazel/runfiles": "^6.3.1", + "jszip": "^3.10.1", + "tmp": "^0.2.3", + "ws": "^8.18.0" }, "engines": { - "node": ">= 14.20.0" + "node": ">= 14.21.0" } }, "node_modules/semver": { @@ -5956,15 +5973,12 @@ } }, "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, "engines": { - "node": ">=8.17.0" + "node": ">=14.14" } }, "node_modules/to-fast-properties": { @@ -6446,9 +6460,9 @@ "dev": true }, "node_modules/ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "engines": { "node": ">=10.0.0" @@ -7073,6 +7087,12 @@ "to-fast-properties": "^2.0.0" } }, + "@bazel/runfiles": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.3.1.tgz", + "integrity": "sha512-1uLNT5NZsUVIGS4syuHwTzZ8HycMPyr6POA3FCE4GbMtc4rhoJk8aZKtNIRthJYfL+iioppi+rTfH3olMPr9nA==", + "dev": true + }, "@ember-data/rfc395-data": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/@ember-data/rfc395-data/-/rfc395-data-0.0.4.tgz", @@ -10750,14 +10770,15 @@ "dev": true }, "selenium-webdriver": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.0.tgz", - "integrity": "sha512-s/HL8WNwy1ggHR244+tAhjhyKMJnZLt1HKJ6Gn7nQgVjB/ybDF+46Uui0qI2J7AjPNJzlUmTncdC/jg/kKkn0A==", + "version": "4.27.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", + "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", "dev": true, "requires": { - "jszip": "^3.10.0", - "tmp": "^0.2.1", - "ws": ">=8.11.0" + "@bazel/runfiles": "^6.3.1", + "jszip": "^3.10.1", + "tmp": "^0.2.3", + "ws": "^8.18.0" } }, "semver": { @@ -11087,13 +11108,10 @@ "dev": true }, "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "dev": true }, "to-fast-properties": { "version": "2.0.0", @@ -11457,9 +11475,9 @@ "dev": true }, "ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "requires": {} }, diff --git a/package.json b/package.json index 22f5a1c1f..59a220303 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,14 @@ "pretty:check": "prettier --check ./", "pretty:fix": "prettier --write ./", "format": "npm run pretty:fix && npm run lint:fix", - "test:chrome": "node tests/run.mjs --browser chrome", - "test:firefox": "node tests/run.mjs --browser firefox", - "test:safari": "node tests/run.mjs --browser safari", - "test:edge": "node tests/run.mjs --browser edge" + "test:chrome": "node tests/run-unittests.mjs --browser chrome", + "test:firefox": "node tests/run-unittests.mjs --browser firefox", + "test:safari": "node tests/run-unittests.mjs --browser safari", + "test:edge": "node tests/run-unittests.mjs --browser edge", + "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome", + "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox", + "test-e2e:safari": "node tests/run-end2end.mjs --browser safari", + "test-e2e:edge": "node tests/run-end2end.mjs --browser edge" }, "devDependencies": { "@babel/core": "^7.21.3", @@ -45,7 +49,7 @@ "http-server": "^14.1.1", "mocha": "^10.2.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0", + "selenium-webdriver": "^4.27.0", "sinon": "^17.0.1", "typescript": "^5.0.4" } diff --git a/resources/main.mjs b/resources/main.mjs index e71d32bd1..588a5294f 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -23,6 +23,7 @@ class MainBenchmarkClient { constructor() { window.addEventListener("DOMContentLoaded", () => this.prepareUI()); this._showSection(window.location.hash); + window.dispatchEvent(new Event("SpeedometerReady")); } start() { @@ -150,6 +151,7 @@ class MainBenchmarkClient { this.showResultsDetails(); else this.showResultsSummary(); + globalThis.dispatchEvent(new Event("SpeedometerDone")); } handleError(error) { @@ -159,6 +161,7 @@ class MainBenchmarkClient { this._metrics = Object.create(null); this._populateInvalidScore(); this.showResultsSummary(); + throw error; } _populateValidScore(scoreResults) { diff --git a/tests/helper.mjs b/tests/helper.mjs new file mode 100644 index 000000000..0c34651b4 --- /dev/null +++ b/tests/helper.mjs @@ -0,0 +1,90 @@ +import commandLineUsage from "command-line-usage"; +import commandLineArgs from "command-line-args"; +import serve from "./server.mjs"; + +import { Builder, Capabilities, logging } from "selenium-webdriver"; + +const optionDefinitions = [ + { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, + { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, + { name: "help", alias: "h", description: "Print this help text." }, +]; + +function printHelp(message = "", exitStatus = 0) { + const usage = commandLineUsage([ + { + header: "Run all tests", + }, + { + header: "Options", + optionList: optionDefinitions, + }, + ]); + if (message) { + console.error(message); + console.error(); + } + console.log(usage); + process.exit(exitStatus); +} + +export default async function testSetup(helpText) { + const options = commandLineArgs(optionDefinitions); + + if ("help" in options) + printHelp(helpText); + + const BROWSER = options?.browser; + if (!BROWSER) + printHelp("No browser specified, use $BROWSER or --browser", 1); + + let capabilities; + switch (BROWSER) { + case "safari": + capabilities = Capabilities.safari(); + break; + + case "firefox": { + capabilities = Capabilities.firefox(); + break; + } + case "chrome": { + capabilities = Capabilities.chrome(); + break; + } + case "edge": { + capabilities = Capabilities.edge(); + break; + } + default: { + printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); + } + } + const prefs = new logging.Preferences(); + prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL); // Capture all log levels + capabilities.setLoggingPrefs(prefs); + + const PORT = options.port; + const server = serve(PORT); + let driver; + + process.on("unhandledRejection", (err) => { + console.error(err); + process.exit(1); + }); + process.once("uncaughtException", (err) => { + console.error(err); + process.exit(1); + }); + process.on("exit", () => stop()); + + driver = await new Builder().withCapabilities(capabilities).build(); + driver.manage().window().setRect({ width: 1200, height: 1000 }); + + function stop() { + server.close(); + if (driver) + driver.close(); + } + return { driver, PORT, stop }; +} diff --git a/tests/index.html b/tests/index.html index c2ff47da8..ecc0ec41a 100644 --- a/tests/index.html +++ b/tests/index.html @@ -27,7 +27,7 @@ }, }); - await import("./benchmark-runner-tests.mjs"); + await import("./unittests/benchmark-runner.mjs"); globalThis.testResults = undefined; globalThis.testRunner = mocha.run(); diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs new file mode 100644 index 000000000..b7e4886cd --- /dev/null +++ b/tests/run-end2end.mjs @@ -0,0 +1,126 @@ +#! /usr/bin/env node + +import assert from "assert"; +import testSetup from "./helper.mjs"; + +import { Suites } from "../resources/tests.mjs"; + +const HELP = ` +This script runs end2end tests by invoking the benchmark via the main +Speedometer page in /index.html. +`.trim(); + +const { driver, PORT, stop } = await testSetup(HELP); + +async function testPage(url) { + console.log(`Testing: ${url}`); + await driver.get(`http://localhost:${PORT}/${url}`); + + await driver.executeAsyncScript((callback) => { + if (globalThis.benchmarkClient) + callback(); + else + globalThis.addEventListener("SpeedometerReady", () => callback(), { once: true }); + }); + + console.log(" - Awaiting Benchmark"); + const { error, metrics } = await driver.executeAsyncScript((callback) => { + globalThis.addEventListener( + "SpeedometerDone", + () => + callback({ + metrics: globalThis.benchmarkClient.metrics, + }), + { once: true } + ); + // Install error handlers to report page errors back to selenium. + globalThis.addEventListener("error", (message, source, lineno, colno, error) => + callback({ + error: { message, source, lineno, colno, error }, + }) + ); + globalThis.addEventListener("unhandledrejection", (e) => { + callback({ + error: { + message: e.reason.toString(), + stack: e.reason?.stack, + }, + }); + }); + globalThis.benchmarkClient.start(); + }); + + if (error) + throw new Error(error.message + (error?.stack ?? "")); + + validateMetrics(metrics); + return metrics; +} + +function validateMetrics(metrics) { + for (const [name, metric] of Object.entries(metrics)) + validateMetric(name, metric); + assert(metrics.Geomean.mean > 0); + assert(metrics.Score.mean > 0); +} + +function validateMetric(name, metric) { + assert(metric.name === name); + assert(metric.mean >= 0); +} + +async function testIterations() { + const iterationCount = 2; + const metrics = await testPage(`index.html?iterationCount=${iterationCount}`); + Suites.forEach((suite) => { + if (!suite.disabled) { + const metric = metrics[suite.name]; + assert(metric.values.length === iterationCount); + } else { + assert(!(suite.name in metrics)); + } + }); + assert(metrics.Geomean.values.length === iterationCount); + assert(metrics.Score.values.length === iterationCount); +} + +async function testAll() { + const metrics = await testPage("index.html?iterationCount=1&tags=all"); + Suites.forEach((suite) => { + assert(suite.name in metrics); + const metric = metrics[suite.name]; + assert(metric.values.length === 1); + }); + assert(metrics.Geomean.values.length === 1); + assert(metrics.Score.values.length === 1); +} + +async function testDeveloperMode() { + const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"]; + const metrics = await testPage(`index.html?${params.join("&")}`); + Suites.forEach((suite) => { + if (suite.name === "Perf-Dashboard") { + const metric = metrics[suite.name]; + assert(metric.values.length === 1); + } else { + assert(!(suite.name in metrics)); + } + }); +} + +async function test() { + try { + await driver.manage().setTimeouts({ script: 60000 }); + await testIterations(); + await testAll(); + await testDeveloperMode(); + console.log("\nTests complete!"); + } catch (e) { + console.error("\nTests failed!"); + throw e; + } finally { + stop(); + } +} + +setImmediate(test); diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs new file mode 100644 index 000000000..5fd7fb86d --- /dev/null +++ b/tests/run-unittests.mjs @@ -0,0 +1,64 @@ +#! /usr/bin/env node + +import assert from "assert"; +import testSetup from "./helper.mjs"; + +const HELP = ` +This script runs the unittests located in tests/unittests/* +through the mocha web interface located at tests/index.html +`.trim(); + +const { driver, PORT, stop } = await testSetup(HELP); + +async function test() { + try { + await driver.get(`http://localhost:${PORT}/tests/index.html`); + + const { testResults, stats } = await driver.executeAsyncScript(function (callback) { + const returnResults = () => + callback({ + stats: globalThis.testRunner.stats, + testResults: globalThis.testResults, + }); + if (window.testResults) + returnResults(); + else + globalThis.addEventListener("test-complete", returnResults, { once: true }); + }); + + printTree(testResults); + + console.log("\nChecking for passed tests..."); + assert(stats.passes > 0); + console.log("Checking for failed tests..."); + assert(stats.failures === 0); + } finally { + console.log("\nTests complete!"); + stop(); + } +} + +function printTree(node) { + console.log(node.title); + + for (const test of node.tests) { + console.group(); + if (test.state === "passed") { + console.log("\x1b[32m✓", `\x1b[0m${test.title}`); + } else { + console.log("\x1b[31m✖", `\x1b[0m${test.title}`); + console.group(); + console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); + console.groupEnd(); + } + console.groupEnd(); + } + + for (const suite of node.suites) { + console.group(); + printTree(suite); + console.groupEnd(); + } +} + +setImmediate(test); diff --git a/tests/run.mjs b/tests/run.mjs deleted file mode 100644 index 8d3461d09..000000000 --- a/tests/run.mjs +++ /dev/null @@ -1,135 +0,0 @@ -#! /usr/bin/env node -/* eslint-disable-next-line no-unused-vars */ -import serve from "./server.mjs"; -import { Builder, Capabilities } from "selenium-webdriver"; -import commandLineArgs from "command-line-args"; -import commandLineUsage from "command-line-usage"; -import assert from "assert"; - -const optionDefinitions = [ - { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, - { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, - { name: "help", alias: "h", description: "Print this help text." }, -]; - -function printHelp(message = "") { - const usage = commandLineUsage([ - { - header: "Run all tests", - }, - { - header: "Options", - optionList: optionDefinitions, - }, - ]); - if (!message) { - console.log(usage); - process.exit(0); - } else { - console.error(message); - console.error(); - console.error(usage); - process.exit(1); - } -} - -const options = commandLineArgs(optionDefinitions); - -if ("help" in options) - printHelp(); - -const BROWSER = options?.browser; -if (!BROWSER) - printHelp("No browser specified, use $BROWSER or --browser"); - -let capabilities; -switch (BROWSER) { - case "safari": - capabilities = Capabilities.safari(); - break; - - case "firefox": { - capabilities = Capabilities.firefox(); - break; - } - case "chrome": { - capabilities = Capabilities.chrome(); - break; - } - case "edge": { - capabilities = Capabilities.edge(); - break; - } - default: { - printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); - } -} - -const PORT = options.port; -const server = serve(PORT); - -let driver; - -function printTree(node) { - console.log(node.title); - - for (const test of node.tests) { - console.group(); - if (test.state === "passed") { - console.log("\x1b[32m✓", `\x1b[0m${test.title}`); - } else { - console.log("\x1b[31m✖", `\x1b[0m${test.title}`); - console.group(); - console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); - console.groupEnd(); - } - console.groupEnd(); - } - - for (const suite of node.suites) { - console.group(); - printTree(suite); - console.groupEnd(); - } -} - -async function test() { - driver = await new Builder().withCapabilities(capabilities).build(); - try { - await driver.get(`http://localhost:${PORT}/tests/index.html`); - - const { testResults, stats } = await driver.executeAsyncScript(function (callback) { - const returnResults = () => - callback({ - stats: globalThis.testRunner.stats, - testResults: globalThis.testResults, - }); - if (window.testResults) - returnResults(); - else - window.addEventListener("test-complete", returnResults, { once: true }); - }); - - printTree(testResults); - - console.log("\nChecking for passed tests..."); - assert(stats.passes > 0); - console.log("Checking for failed tests..."); - assert(stats.failures === 0); - } finally { - console.log("\nTests complete!"); - driver.quit(); - server.close(); - } -} - -process.on("unhandledRejection", (err) => { - console.error(err); - process.exit(1); -}); -process.once("uncaughtException", (err) => { - console.error(err); - process.exit(1); -}); - -setImmediate(test); diff --git a/tests/server.mjs b/tests/server.mjs index 0b209b897..8f73e7067 100644 --- a/tests/server.mjs +++ b/tests/server.mjs @@ -23,8 +23,8 @@ export default function serve(port) { throw new Error("Port is required"); const prepareFile = async (url) => { - const paths = [STATIC_PATH, url]; - if (url.endsWith("/")) + const paths = [STATIC_PATH, url.pathname]; + if (url.pathname.endsWith("/")) paths.push("index.html"); const filePath = path.join(...paths); const pathTraversal = !filePath.startsWith(STATIC_PATH); @@ -38,7 +38,8 @@ export default function serve(port) { const server = http .createServer(async (req, res) => { - const file = await prepareFile(req.url); + const url = new URL(`http://localhost${req.url}`); + const file = await prepareFile(url); const statusCode = file.found ? 200 : 404; const mimeType = MIME_TYPES[file.ext] || MIME_TYPES.default; res.writeHead(statusCode, { "Content-Type": mimeType }); diff --git a/tests/benchmark-runner-tests.mjs b/tests/unittests/benchmark-runner.mjs similarity index 97% rename from tests/benchmark-runner-tests.mjs rename to tests/unittests/benchmark-runner.mjs index a2d5e51bb..1dec9c93a 100644 --- a/tests/benchmark-runner-tests.mjs +++ b/tests/unittests/benchmark-runner.mjs @@ -1,7 +1,7 @@ -import { BenchmarkRunner } from "../resources/benchmark-runner.mjs"; -import { SuiteRunner } from "../resources/suite-runner.mjs"; -import { TestRunner } from "../resources/shared/test-runner.mjs"; -import { defaultParams } from "../resources/shared/params.mjs"; +import { BenchmarkRunner } from "../../resources/benchmark-runner.mjs"; +import { SuiteRunner } from "../../resources/suite-runner.mjs"; +import { TestRunner } from "../../resources/shared/test-runner.mjs"; +import { defaultParams } from "../../resources/shared/params.mjs"; function TEST_FIXTURE(name) { return {