Implement end2end tests

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: CI
+
+on:
+    push:
+        branches: ["main"]
+    pull_request:
+        branches: ["main"]
+
+    # Allows you to run this workflow manually from the Actions tab
+    workflow_dispatch:
+
+jobs:
+    build:
+        name: Build
+        runs-on: macos-latest
+        strategy:
+            matrix:
+                browser: [chrome, firefox, safari]
+        steps:
+            - name: Install Firefox
+              if: ${{ matrix.browser == 'firefox' }}
+              run: brew install --cask firefox
+            - name: Checkout Branch
+              uses: actions/checkout@v3
+            - name: Setup Node
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 18.13.0
+            - name: Install
+              run: npm install
+            - name: Run tests
+              run: |
+                  echo "Running in $BROWSER"
+                  npm run test:${{ matrix.browser }}

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .directory
+/node_modules
 
 # Ignore auto-generated files by VS & VSCode.
 /.vs/

JetStreamDriver.js

@@ -27,35 +27,47 @@
 
 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
 
+const defaultIterationCount = 120;
+const defaultWorstCaseCount = 4;
+
 globalThis.performance ??= Date;
 globalThis.RAMification ??= false;
 globalThis.testIterationCount ??= undefined;
 globalThis.testIterationCountMap ??= new Map();
+globalThis.testWorstCaseCount ??= undefined;
 globalThis.testWorstCaseCountMap ??= new Map();
 globalThis.dumpJSONResults ??= false;
 globalThis.customTestList ??= [];
+globalThis.startDelay ??= undefined;
 
 let shouldReport = false;
-let startDelay;
+
+function getIntParam(urlParams, key) {
+    if (!urlParams.has(key))
+        return undefined
+    const rawValue = urlParams.get(key);
+    const value = parseInt(rawValue);
+    if (value <= 0)
+        throw new Error(`Expected positive value for ${key}, but got ${rawValue}`)
+    return value
+}
+
 if (typeof(URLSearchParams) !== "undefined") {
     const urlParameters = new URLSearchParams(window.location.search);
     shouldReport = urlParameters.has('report') && urlParameters.get('report').toLowerCase() == 'true';
     if (shouldReport)
-        startDelay = 4000;
-    if (urlParameters.has('startDelay'))
-        startDelay = urlParameters.get('startDelay');
+        globalThis.startDelay = 4000;
     if (urlParameters.has('test'))
         customTestList = urlParameters.getAll("test");
-
+    globalThis.startDelay = getIntParam(urlParameters, "startDelay");
+    globalThis.testIterationCount = getIntParam(urlParameters, "iterationCount");
+    globalThis.testWorstCaseCount = getIntParam(urlParameters, "worstCaseCount");
 }
 
 // Used for the promise representing the current benchmark run.
 this.currentResolve = null;
 this.currentReject = null;
 
-const defaultIterationCount = 120;
-const defaultWorstCaseCount = 4;
-
 let showScoreDetails = false;
 let categoryScores = null;
 
@@ -83,6 +95,8 @@ function getIterationCount(plan) {
 function getWorstCaseCount(plan) {
     if (testWorstCaseCountMap.has(plan.name))
         return testWorstCaseCountMap.get(plan.name);
+    if (testWorstCaseCount)
+        return testWorstCaseCount;
     if (plan.worstCaseCount)
         return plan.worstCaseCount;
     return defaultWorstCaseCount;
@@ -220,6 +234,7 @@ const fileLoader = (function() {
 
 class Driver {
     constructor() {
+        this.isReady = false;
         this.benchmarks = [];
         this.blobDataCache = { };
         this.loadCache = { };
@@ -314,6 +329,11 @@ class Driver {
 
         this.reportScoreToRunBenchmarkRunner();
         this.dumpJSONResultsIfNeeded();
+        if (isInBrowser) {
+            globalThis.dispatchEvent(new CustomEvent("JetStreamDone", {
+                detail: this.resultsObject()
+            }));
+        }
     }
 
     runCode(string)
@@ -426,8 +446,12 @@ class Driver {
         await this.prefetchResourcesForBrowser();
         await this.fetchResources();
         this.prepareToRun();
-        if (isInBrowser && startDelay !== undefined) {
-            setTimeout(() => this.start(), startDelay);
+        this.isReady = true;
+        if (isInBrowser) {
+            globalThis.dispatchEvent(new Event("JetStreamReady"));
+            if (shouldReport) {
+                setTimeout(() => this.start(), globalThis.startDelay);
+            }
         }
     }
 
@@ -479,7 +503,7 @@ class Driver {
         }
     }
 
-    resultsJSON()
+    resultsObject()
     {
         const results = {};
         for (const benchmark of this.benchmarks) {
@@ -498,8 +522,13 @@ class Driver {
         }
 
         results = {"JetStream3.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
+        return results;
+
+    }
 
-        return JSON.stringify(results);
+    resultsJSON()
+    {
+        return JSON.stringify(this.resultsObject());
     }
 
     dumpJSONResultsIfNeeded()
@@ -564,7 +593,7 @@ class Benchmark {
                 results.push(Math.max(1, end - start));
             }
             if (__benchmark.validate)
-                __benchmark.validate();
+                __benchmark.validate(${this.iterations});
             top.currentResolve(results);`;
     }
 
@@ -1006,7 +1035,7 @@ class AsyncBenchmark extends DefaultBenchmark {
                 results.push(Math.max(1, end - start));
             }
             if (__benchmark.validate)
-                __benchmark.validate();
+                __benchmark.validate(${this.iterations});
             top.currentResolve(results);
         }
         doRun().catch((error) => { top.currentReject(error); });`

RexBench/FlightPlanner/benchmark.js

@@ -37,7 +37,7 @@ class Benchmark {
         }
     }
 
-    validate()
+    validate(iterations)
     {
         for (let flightPlan of expectedFlightPlans) {
             flightPlan.calculate();

RexBench/OfflineAssembler/benchmark.js

@@ -36,7 +36,7 @@ class Benchmark {
         this.ast = parse("LowLevelInterpreter.asm");
     }
 
-    validate()
+    validate(iterations)
     {
         let astDumpedAsLines = this.ast.dump().split("\n");
 

RexBench/UniPoker/benchmark.js

@@ -39,8 +39,11 @@ class Benchmark {
         playHands(this._players);
     }
 
-    validate()
+    validate(iterations)
     {
+        if (!iterations)
+            throw "Invalid iterations"
+        const playerExpectations = getPlayerExpectations(iterations)
         if (this._players.length != playerExpectations.length)
             throw "Expect " + playerExpectations.length + ", but actually have " + this._players.length;
         if (isInBrowser) {

RexBench/UniPoker/expected.js

@@ -24,12 +24,15 @@
  */
 "use strict";
 
+const defaultIterationCount = 120;
+
 class PlayerExpectation
 {
-    constructor(wins, handTypeCounts)
+    constructor(iterations, wins, handTypeCounts)
     {
-        this._wins = wins;
-        this._handTypeCounts = handTypeCounts;
+        const factor =  iterations / defaultIterationCount;
+        this._wins = wins * factor;
+        this._handTypeCounts = handTypeCounts.map(each => each * factor);
     }
 
     validate(player)
@@ -62,10 +65,13 @@ PlayerExpectation._handTypes = [
     "Straight Flushes",
     "Royal Flushes"
 ];
-
-var playerExpectations = [];
 
-playerExpectations.push(new PlayerExpectation(60120, [120600, 102000, 10080, 5040, 1440, 360, 480, 0, 0, 0]));
-playerExpectations.push(new PlayerExpectation(60480, [120360, 99600, 11760, 6120, 1200, 360, 480, 120, 0, 0]));
-playerExpectations.push(new PlayerExpectation(61440, [121200, 99720, 11040, 6120, 1080, 480, 360, 0, 0, 0]));
-playerExpectations.push(new PlayerExpectation(57960, [121320, 100440, 11040, 5760, 840, 480, 120, 0, 0, 0]));
+
+function getPlayerExpectations(iterations) {
+    const playerExpectations = [];
+    playerExpectations.push(new PlayerExpectation(iterations, 60120, [120600, 102000, 10080, 5040, 1440, 360, 480, 0, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 60480, [120360, 99600, 11760, 6120, 1200, 360, 480, 120, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 61440, [121200, 99720, 11040, 6120, 1080, 480, 360, 0, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 57960, [121320, 100440, 11040, 5760, 840, 480, 120, 0, 0, 0]));
+    return playerExpectations;
+}

bigint/bigdenary-benchmark.js

@@ -53,7 +53,7 @@ class Benchmark {
             areFirstAndLastResultsEqual(() => bd1.dividedBy(bd2));
     }
 
-    validate() {
+    validate(iterations) {
         if (!this._allFirstAndLastResultsAreEqual)
             throw new Error("Expected all first and last results to be equal, but they aren't.");
     }

bigint/paillier-benchmark.js

@@ -52,7 +52,7 @@ class Benchmark {
         this._decryptedMulIsCorrect &&= decryptedMul === 123456789012345678900n;
     }
 
-    validate() {
+    validate(iterations) {
         if (!this._c1DecryptedIsCorrect)
             throw new Error("Bad value: c1Decrypted!");
 

generators/js-tokens.js

@@ -451,7 +451,7 @@ class Benchmark {
         }
     }
 
-    validate() {
+    validate(iterations) {
         if (this.tokenCount !== 113975)
             throw new Error(`this.tokenCount of ${this.tokenCount} is invalid!`);
     }

generators/lazy-collections.js

@@ -613,7 +613,7 @@ class Benchmark {
     this.totalLength += binaryTree().length;
   }
 
-  validate() {
+  validate(iterations) {
     if (this.totalLength !== 635)
       throw new Error(`this.totalLength of ${this.totalLength} is invalid!`);
   }