fixing metrics

WebKit · Jan 13, 2025 · 85fa36d · 85fa36d
1 parent fe2662a
commit 85fa36d
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 19 deletions.
diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs
@@ -439,10 +439,10 @@ export class BenchmarkRunner {
 
     _appendIterationMetrics() {
         const getMetric = (name, unit = "ms") => this._metrics[name] || (this._metrics[name] = new Metric(name, unit));
-        const iterationTotalMetric = (i) => {
+        const iterationMetric = (i, name) => {
             if (i >= params.iterationCount)
                 throw new Error(`Requested iteration=${i} does not exist.`);
-            return getMetric(`Iteration-${i}-Total`);
+            return getMetric(`Iteration-${i}-${name}`);
         };
 
         const collectSubMetrics = (prefix, items, parent) => {
@@ -467,25 +467,27 @@ export class BenchmarkRunner {
             // Prepare all iteration metrics so they are listed at the end of
             // of the _metrics object, before "Total" and "Score".
             for (let i = 0; i < this._iterationCount; i++)
-                iterationTotalMetric(i).description = `Test totals for iteration ${i}`;
+                iterationMetric(i, "Total").description = `Test totals for iteration ${i}`;
             getMetric("Geomean", "ms").description = "Geomean of test totals";
             getMetric("Score", "score").description = "Scaled inverse of the Geomean";
             if (params.measurePrepare)
                 getMetric("Prepare", "ms").description = "Geomean of workload prepare times";
         }
 
         const geomean = getMetric("Geomean");
-        const iterationTotal = iterationTotalMetric(geomean.length);
+        const iteration = geomean.length;
+        const iterationTotal = iterationMetric(iteration, "Total");
         for (const results of Object.values(iterationResults))
             iterationTotal.add(results.total);
         iterationTotal.computeAggregatedMetrics();
         geomean.add(iterationTotal.geomean);
         getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
 
         if (params.measurePrepare) {
-            const iterationPrepare = new Metric("tmp");
+            const iterationPrepare = iterationMetric(iteration, "Prepare");
             for (const results of Object.values(iterationResults))
-                iterationPrepare.add(results.totalPrepare);
+                iterationPrepare.add(results.prepare);
+            iterationPrepare.computeAggregatedMetrics();
             const prepare = getMetric("Prepare");
             prepare.add(iterationPrepare.geomean);
         }

diff --git a/resources/main.mjs b/resources/main.mjs
@@ -248,8 +248,8 @@ class MainBenchmarkClient {
         const plotWidth = (params.viewport.width - 120) / 2;
         const aggregateMetrics = [metrics.Geomean];
         if (params.measurePrepare) {
-            aggregateMetrics.append(metrics.Prepare);
-            console.log(metrics.Prepare)
+            aggregateMetrics.push(metrics.Prepare);
+            console.log(metrics.Prepare);
         }
         document.getElementById("aggregate-chart").innerHTML = renderMetricView({
             metrics: aggregateMetrics,

diff --git a/resources/shared/benchmark.mjs b/resources/shared/benchmark.mjs
@@ -44,6 +44,7 @@ export class BenchmarkSuite {
     async runAndRecord(params, onProgress) {
         const measuredValues = {
             tests: {},
+            prepare: 0,
             total: 0,
         };
         const suiteStartLabel = `suite-${this.name}-start`;

diff --git a/resources/suite-runner.mjs b/resources/suite-runner.mjs
@@ -14,7 +14,7 @@ export class SuiteRunner {
         // FIXME: Create SuiteRunner-local measuredValues.
         this.#suiteResults = measuredValues.tests[suite.name];
         if (!this.#suiteResults) {
-            this.#suiteResults = { tests: {}, total: 0 };
+            this.#suiteResults = { tests: {}, prepare: 0, total: 0 };
             measuredValues.tests[suite.name] = this.#suiteResults;
         }
         this.#frame = frame;
@@ -83,17 +83,19 @@ export class SuiteRunner {
         performance.mark(suiteEndLabel);
 
         performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
-        this._validateSuiteTotal();
+        this._validateSuiteResults();
         await this._updateClient();
     }
 
-    _validateSuiteTotal() {
+    _validateSuiteResults() {
         // When the test is fast and the precision is low (for example with Firefox'
         // privacy.resistFingerprinting preference), it's possible that the measured
         // total duration for an entire is 0.
-        const suiteTotal = this.#suiteResults.total;
+        const { suiteTotal, suitePrepare } = this.#suiteResults.total;
         if (suiteTotal === 0)
             throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
+        if (this.#params.measurePrepare && suitePrepare === 0)
+            throw new Error(`Got invalid 0-time prepare time for suite ${this.#suite.name}: ${suitePrepare}`);
     }
 
     async _loadFrame() {
@@ -111,10 +113,12 @@ export class SuiteRunner {
             return;
 
         let total = syncTime + asyncTime;
-        if (this.#params.measurePrepare)
-            total += this.#prepareTime;
-        this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime, Prepare: this.#prepareTime }, total: total };
-        this.#suiteResults.total += total;
+        this.#suiteResults.tests[test.name] = {
+            tests: { Sync: syncTime, Async: asyncTime},
+            total: total
+        };
+        this.#suiteResults.prepare = this.#prepareTime;
+        this.#suiteResults.total = total;
     };
 
     async _updateClient(suite = this.#suite) {
@@ -125,6 +129,7 @@ export class SuiteRunner {
 
 export class RemoteSuiteRunner extends SuiteRunner {
     #appId;
+    #prepareTime;
 
     get appId() {
         return this.#appId;
@@ -165,7 +170,8 @@ export class RemoteSuiteRunner extends SuiteRunner {
 
         performance.mark(suitePrepareEndLabel);
 
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        const entry = performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+        this.#prepareTime = entry.duration;
     }
 
     async _runSuite() {
@@ -179,9 +185,10 @@ export class RemoteSuiteRunner extends SuiteRunner {
             ...response.result.tests,
         };
 
-        this.suiteResults.total += response.result.total;
+        this.suiteResults.prepare = this.#prepareTime;
+        this.suiteResults.total = response.result.total;
 
-        this._validateSuiteTotal();
+        this._validateSuiteResults();
         await this._updateClient();
     }