From 8b101cd5819ad73f66227f45d78894966d6a9797 Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Mon, 22 Apr 2024 13:07:53 +0200
Subject: [PATCH 1/6] Calculate and report variance in performance results

---
 .../config/performance-reporter.ts            | 57 +++++++++++++++----
 test/performance/utils.js                     |  9 +++
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts
index b449e36540404e..e057c272654c52 100644
--- a/test/performance/config/performance-reporter.ts
+++ b/test/performance/config/performance-reporter.ts
@@ -13,7 +13,7 @@ import type {
 /**
  * Internal dependencies
  */
-import { average, median, round } from '../utils';
+import { average, median, variance, round } from '../utils';
 
 export interface WPRawPerformanceResults {
 	timeToFirstByte: number[];
@@ -45,24 +45,43 @@ export interface WPRawPerformanceResults {
 
 export interface WPPerformanceResults {
 	timeToFirstByte?: number;
+	timeToFirstByteV?: number;
 	largestContentfulPaint?: number;
+	largestContentfulPaintV?: number;
 	lcpMinusTtfb?: number;
+	lcpMinusTtfbV?: number;
 	serverResponse?: number;
+	serverResponseV?: number;
 	firstPaint?: number;
+	firstPaintV?: number;
 	domContentLoaded?: number;
+	domContentLoadedV?: number;
 	loaded?: number;
+	loadedV?: number;
 	firstContentfulPaint?: number;
+	firstContentfulPaintV?: number;
 	firstBlock?: number;
+	firstBlockV?: number;
 	type?: number;
+	typeV?: number;
 	typeWithoutInspector?: number;
+	typeWithoutInspectorV?: number;
 	typeWithTopToolbar?: number;
+	typeWithTopToolbarV?: number;
 	typeContainer?: number;
+	typeContainerV?: number;
 	focus?: number;
+	focusV?: number;
 	inserterOpen?: number;
+	inserterOpenV?: number;
 	inserterSearch?: number;
+	inserterSearchV?: number;
 	inserterHover?: number;
+	inserterHoverV?: number;
 	loadPatterns?: number;
+	loadPatternsV?: number;
 	listViewOpen?: number;
+	listViewOpenV?: number;
 	navigate?: number;
 	wpBeforeTemplate?: number;
 	wpTemplate?: number;
@@ -82,24 +101,42 @@ export function curateResults(
 	results: WPRawPerformanceResults
 ): WPPerformanceResults {
 	const output = {
-		timeToFirstByte: median( results.timeToFirstByte ),
-		largestContentfulPaint: median( results.largestContentfulPaint ),
-		lcpMinusTtfb: median( results.lcpMinusTtfb ),
-		serverResponse: median( results.serverResponse ),
-		firstPaint: median( results.firstPaint ),
-		domContentLoaded: median( results.domContentLoaded ),
-		loaded: median( results.loaded ),
-		firstContentfulPaint: median( results.firstContentfulPaint ),
-		firstBlock: median( results.firstBlock ),
+		timeToFirstByte: average( results.timeToFirstByte ),
+		timeToFirstByteV: variance( results.timeToFirstByte ),
+		largestContentfulPaint: average( results.largestContentfulPaint ),
+		largestContentfulPaintV: variance( results.largestContentfulPaint ),
+		lcpMinusTtfb: average( results.lcpMinusTtfb ),
+		lcpMinusTtfbV: variance( results.lcpMinusTtfb ),
+		serverResponse: average( results.serverResponse ),
+		serverResponseV: variance( results.serverResponse ),
+		firstPaint: average( results.firstPaint ),
+		firstPaintV: variance( results.firstPaint ),
+		domContentLoaded: average( results.domContentLoaded ),
+		domContentLoadedV: variance( results.domContentLoaded ),
+		loaded: average( results.loaded ),
+		loadedV: variance( results.loaded ),
+		firstContentfulPaint: average( results.firstContentfulPaint ),
+		firstContentfulPaintV: variance( results.firstContentfulPaint ),
+		firstBlock: average( results.firstBlock ),
+		firstBlockV: variance( results.firstBlock ),
 		type: average( results.type ),
+		typeV: variance( results.type ),
 		typeWithoutInspector: average( results.typeWithoutInspector ),
+		typeWithoutInspectorV: variance( results.typeWithoutInspector ),
 		typeWithTopToolbar: average( results.typeWithTopToolbar ),
+		typeWithTopToolbarV: variance( results.typeWithTopToolbar ),
 		typeContainer: average( results.typeContainer ),
+		typeContainerV: variance( results.typeContainer ),
 		focus: average( results.focus ),
+		focusV: variance( results.focus ),
 		inserterOpen: average( results.inserterOpen ),
+		inserterOpenV: variance( results.inserterOpen ),
 		inserterSearch: average( results.inserterSearch ),
+		inserterSearchV: variance( results.inserterSearch ),
 		inserterHover: average( results.inserterHover ),
+		inserterHoverV: variance( results.inserterHover ),
 		loadPatterns: average( results.loadPatterns ),
+		loadPatternsV: variance( results.loadPatterns ),
 		listViewOpen: average( results.listViewOpen ),
 		navigate: median( results.navigate ),
 		wpBeforeTemplate: median( results.wpBeforeTemplate ),
diff --git a/test/performance/utils.js b/test/performance/utils.js
index 72a851a4ffabc6..31dadccbeb3288 100644
--- a/test/performance/utils.js
+++ b/test/performance/utils.js
@@ -19,6 +19,15 @@ export function average( array ) {
 	return sum( array ) / array.length;
 }
 
+export function variance( array ) {
+	if ( ! array || ! array.length ) return undefined;
+
+	return Math.sqrt(
+		sum( array.map( ( x ) => x ** 2 ) ) / array.length -
+			( sum( array ) / array.length ) ** 2
+	);
+}
+
 export function median( array ) {
 	if ( ! array || ! array.length ) {
 		return undefined;

From 036bdfa139b036ee3d614d9a64ea3dd84f592998 Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Fri, 10 May 2024 10:43:53 +0200
Subject: [PATCH 2/6] Calculate quartiles and outliers, report in table

---
 .../config/performance-reporter.ts            | 124 ++++++++++--------
 test/performance/utils.js                     |  39 +++++-
 2 files changed, 107 insertions(+), 56 deletions(-)

diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts
index e057c272654c52..55e3425fe494a4 100644
--- a/test/performance/config/performance-reporter.ts
+++ b/test/performance/config/performance-reporter.ts
@@ -13,7 +13,7 @@ import type {
 /**
  * Internal dependencies
  */
-import { average, median, variance, round } from '../utils';
+import { quartiles, round } from '../utils';
 
 export interface WPRawPerformanceResults {
 	timeToFirstByte: number[];
@@ -90,6 +90,25 @@ export interface WPPerformanceResults {
 	wpDbQueries?: number;
 }
 
+function stats( values: number[] ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const iqr = q75 - q25;
+	const out = values.filter(
+		( v ) => v > q75 + 1.5 * iqr || v < q25 - 1.5 * iqr
+	);
+	const cnt = values.length;
+	return {
+		q25: round( q25 ),
+		q50: round( q50 ),
+		q75: round( q75 ),
+		out: out.map( ( n ) => round( n ) ),
+		cnt,
+	};
+}
+
 /**
  * Curate the raw performance results.
  *
@@ -101,62 +120,37 @@ export function curateResults(
 	results: WPRawPerformanceResults
 ): WPPerformanceResults {
 	const output = {
-		timeToFirstByte: average( results.timeToFirstByte ),
-		timeToFirstByteV: variance( results.timeToFirstByte ),
-		largestContentfulPaint: average( results.largestContentfulPaint ),
-		largestContentfulPaintV: variance( results.largestContentfulPaint ),
-		lcpMinusTtfb: average( results.lcpMinusTtfb ),
-		lcpMinusTtfbV: variance( results.lcpMinusTtfb ),
-		serverResponse: average( results.serverResponse ),
-		serverResponseV: variance( results.serverResponse ),
-		firstPaint: average( results.firstPaint ),
-		firstPaintV: variance( results.firstPaint ),
-		domContentLoaded: average( results.domContentLoaded ),
-		domContentLoadedV: variance( results.domContentLoaded ),
-		loaded: average( results.loaded ),
-		loadedV: variance( results.loaded ),
-		firstContentfulPaint: average( results.firstContentfulPaint ),
-		firstContentfulPaintV: variance( results.firstContentfulPaint ),
-		firstBlock: average( results.firstBlock ),
-		firstBlockV: variance( results.firstBlock ),
-		type: average( results.type ),
-		typeV: variance( results.type ),
-		typeWithoutInspector: average( results.typeWithoutInspector ),
-		typeWithoutInspectorV: variance( results.typeWithoutInspector ),
-		typeWithTopToolbar: average( results.typeWithTopToolbar ),
-		typeWithTopToolbarV: variance( results.typeWithTopToolbar ),
-		typeContainer: average( results.typeContainer ),
-		typeContainerV: variance( results.typeContainer ),
-		focus: average( results.focus ),
-		focusV: variance( results.focus ),
-		inserterOpen: average( results.inserterOpen ),
-		inserterOpenV: variance( results.inserterOpen ),
-		inserterSearch: average( results.inserterSearch ),
-		inserterSearchV: variance( results.inserterSearch ),
-		inserterHover: average( results.inserterHover ),
-		inserterHoverV: variance( results.inserterHover ),
-		loadPatterns: average( results.loadPatterns ),
-		loadPatternsV: variance( results.loadPatterns ),
-		listViewOpen: average( results.listViewOpen ),
-		navigate: median( results.navigate ),
-		wpBeforeTemplate: median( results.wpBeforeTemplate ),
-		wpTemplate: median( results.wpTemplate ),
-		wpTotal: median( results.wpTotal ),
-		wpMemoryUsage: median( results.wpMemoryUsage ),
-		wpDbQueries: median( results.wpDbQueries ),
+		timeToFirstByte: stats( results.timeToFirstByte ),
+		largestContentfulPaint: stats( results.largestContentfulPaint ),
+		lcpMinusTtfb: stats( results.lcpMinusTtfb ),
+		serverResponse: stats( results.serverResponse ),
+		firstPaint: stats( results.firstPaint ),
+		domContentLoaded: stats( results.domContentLoaded ),
+		loaded: stats( results.loaded ),
+		firstContentfulPaint: stats( results.firstContentfulPaint ),
+		firstBlock: stats( results.firstBlock ),
+		type: stats( results.type ),
+		typeWithoutInspector: stats( results.typeWithoutInspector ),
+		typeWithTopToolbar: stats( results.typeWithTopToolbar ),
+		typeContainer: stats( results.typeContainer ),
+		focus: stats( results.focus ),
+		inserterOpen: stats( results.inserterOpen ),
+		inserterSearch: stats( results.inserterSearch ),
+		inserterHover: stats( results.inserterHover ),
+		loadPatterns: stats( results.loadPatterns ),
+		listViewOpen: stats( results.listViewOpen ),
+		navigate: stats( results.navigate ),
+		wpBeforeTemplate: stats( results.wpBeforeTemplate ),
+		wpTemplate: stats( results.wpTemplate ),
+		wpTotal: stats( results.wpTotal ),
+		wpMemoryUsage: stats( results.wpMemoryUsage ),
+		wpDbQueries: stats( results.wpDbQueries ),
 	};
 
-	return (
+	return Object.fromEntries(
 		Object.entries( output )
 			// Reduce the output to contain taken metrics only.
-			.filter( ( [ _, value ] ) => typeof value === 'number' )
-			.reduce(
-				( acc, [ key, value ] ) => ( {
-					...acc,
-					[ key ]: round( value ),
-				} ),
-				{}
-			)
+			.filter( ( [ _, value ] ) => value !== undefined )
 	);
 }
 class PerformanceReporter implements Reporter {
@@ -192,13 +186,21 @@ class PerformanceReporter implements Reporter {
 
 			const curatedResults = curateResults( JSON.parse( resultsBody ) );
 
+			// For now, to keep back compat, save only the medians, not the full stats.
+			const savedResults = Object.fromEntries(
+				Object.entries( curatedResults ).map( ( [ key, value ] ) => [
+					key,
+					value.q50,
+				] )
+			);
+
 			// Save curated results to file.
 			writeFileSync(
 				path.join(
 					resultsPath,
 					`${ resultsId }.performance-results.json`
 				),
-				JSON.stringify( curatedResults, null, 2 )
+				JSON.stringify( savedResults, null, 2 )
 			);
 
 			this.results[ testSuite ] = curatedResults;
@@ -219,7 +221,19 @@ class PerformanceReporter implements Reporter {
 			const printableResults: Record< string, { value: string } > = {};
 
 			for ( const [ key, value ] of Object.entries( results ) ) {
-				printableResults[ key ] = { value: `${ value } ms` };
+				const p = value.q75 - value.q50;
+				const pp = round( ( 100 * p ) / value.q50 );
+				const m = value.q50 - value.q25;
+				const mp = round( ( 100 * m ) / value.q50 );
+				const outs =
+					value.out.length > 0
+						? ' [' + value.out.join( ', ' ) + ']'
+						: '';
+				printableResults[ key ] = {
+					value: `${ value.q50 } ±${ round( p ) }/${ round(
+						m
+					) } ms (±${ pp }/${ mp }%)${ outs } (${ value.cnt })`,
+				};
 			}
 
 			// eslint-disable-next-line no-console
diff --git a/test/performance/utils.js b/test/performance/utils.js
index 31dadccbeb3288..2128b49d9e7925 100644
--- a/test/performance/utils.js
+++ b/test/performance/utils.js
@@ -20,7 +20,9 @@ export function average( array ) {
 }
 
 export function variance( array ) {
-	if ( ! array || ! array.length ) return undefined;
+	if ( ! array || ! array.length ) {
+		return undefined;
+	}
 
 	return Math.sqrt(
 		sum( array.map( ( x ) => x ** 2 ) ) / array.length -
@@ -42,6 +44,41 @@ export function median( array ) {
 	return numbers[ middleIndex ];
 }
 
+export function quartiles( array ) {
+	const numbers = array.slice().sort( ( a, b ) => a - b );
+
+	function med( offset, length ) {
+		if ( length % 2 === 0 ) {
+			// even length, average of two middle numbers
+			return (
+				( numbers[ offset + length / 2 - 1 ] +
+					numbers[ offset + length / 2 ] ) /
+				2
+			);
+		}
+
+		// odd length, exact middle point
+		return numbers[ offset + ( length - 1 ) / 2 ];
+	}
+
+	const q50 = med( 0, numbers.length );
+
+	let q25, q75;
+	if ( numbers.length % 2 === 0 ) {
+		// medians of two exact halves
+		const mid = numbers.length / 2;
+		q25 = med( 0, mid );
+		q75 = med( mid, mid );
+	} else {
+		// quartiles are average of medians of the smaller and bigger slice
+		const midl = ( numbers.length - 1 ) / 2;
+		const midh = ( numbers.length + 1 ) / 2;
+		q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2;
+		q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2;
+	}
+	return { q25, q50, q75 };
+}
+
 export function minimum( array ) {
 	if ( ! array || ! array.length ) {
 		return undefined;

From 0420a9d265f4ce5f7f2626e25652826f4da49fc3 Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Fri, 10 May 2024 13:46:22 +0200
Subject: [PATCH 3/6] Fixup types

---
 .../config/performance-reporter.ts            | 84 ++++++++-----------
 1 file changed, 36 insertions(+), 48 deletions(-)

diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts
index 55e3425fe494a4..779013684a0736 100644
--- a/test/performance/config/performance-reporter.ts
+++ b/test/performance/config/performance-reporter.ts
@@ -43,54 +43,43 @@ export interface WPRawPerformanceResults {
 	wpDbQueries: number[];
 }
 
+type PerformanceStats = {
+	q25: number;
+	q50: number;
+	q75: number;
+	out: number[]; // outliers
+	cnt: number; // number of data points
+};
+
 export interface WPPerformanceResults {
-	timeToFirstByte?: number;
-	timeToFirstByteV?: number;
-	largestContentfulPaint?: number;
-	largestContentfulPaintV?: number;
-	lcpMinusTtfb?: number;
-	lcpMinusTtfbV?: number;
-	serverResponse?: number;
-	serverResponseV?: number;
-	firstPaint?: number;
-	firstPaintV?: number;
-	domContentLoaded?: number;
-	domContentLoadedV?: number;
-	loaded?: number;
-	loadedV?: number;
-	firstContentfulPaint?: number;
-	firstContentfulPaintV?: number;
-	firstBlock?: number;
-	firstBlockV?: number;
-	type?: number;
-	typeV?: number;
-	typeWithoutInspector?: number;
-	typeWithoutInspectorV?: number;
-	typeWithTopToolbar?: number;
-	typeWithTopToolbarV?: number;
-	typeContainer?: number;
-	typeContainerV?: number;
-	focus?: number;
-	focusV?: number;
-	inserterOpen?: number;
-	inserterOpenV?: number;
-	inserterSearch?: number;
-	inserterSearchV?: number;
-	inserterHover?: number;
-	inserterHoverV?: number;
-	loadPatterns?: number;
-	loadPatternsV?: number;
-	listViewOpen?: number;
-	listViewOpenV?: number;
-	navigate?: number;
-	wpBeforeTemplate?: number;
-	wpTemplate?: number;
-	wpTotal?: number;
-	wpMemoryUsage?: number;
-	wpDbQueries?: number;
+	timeToFirstByte?: PerformanceStats;
+	largestContentfulPaint?: PerformanceStats;
+	lcpMinusTtfb?: PerformanceStats;
+	serverResponse?: PerformanceStats;
+	firstPaint?: PerformanceStats;
+	domContentLoaded?: PerformanceStats;
+	loaded?: PerformanceStats;
+	firstContentfulPaint?: PerformanceStats;
+	firstBlock?: PerformanceStats;
+	type?: PerformanceStats;
+	typeWithoutInspector?: PerformanceStats;
+	typeWithTopToolbar?: PerformanceStats;
+	typeContainer?: PerformanceStats;
+	focus?: PerformanceStats;
+	inserterOpen?: PerformanceStats;
+	inserterSearch?: PerformanceStats;
+	inserterHover?: PerformanceStats;
+	loadPatterns?: PerformanceStats;
+	listViewOpen?: PerformanceStats;
+	navigate?: PerformanceStats;
+	wpBeforeTemplate?: PerformanceStats;
+	wpTemplate?: PerformanceStats;
+	wpTotal?: PerformanceStats;
+	wpMemoryUsage?: PerformanceStats;
+	wpDbQueries?: PerformanceStats;
 }
 
-function stats( values: number[] ) {
+function stats( values: number[] ): PerformanceStats | undefined {
 	if ( ! values || values.length === 0 ) {
 		return undefined;
 	}
@@ -112,9 +101,8 @@ function stats( values: number[] ) {
 /**
  * Curate the raw performance results.
  *
- * @param {WPRawPerformanceResults} results
- *
- * @return {WPPerformanceResults} Curated Performance results.
+ * @param results Raw results.
+ * @return Curated statistics for the results.
  */
 export function curateResults(
 	results: WPRawPerformanceResults

From 0529b721e7ffae4eb59082929eaa86847553be7c Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Fri, 10 May 2024 17:07:09 +0200
Subject: [PATCH 4/6] Calculate stats in CI, too

---
 bin/log-performance-results.js                |   4 +-
 bin/plugin/commands/performance.js            | 164 ++++++++++++------
 .../config/performance-reporter.ts            |  48 +----
 test/performance/utils.js                     |  14 ++
 4 files changed, 135 insertions(+), 95 deletions(-)

diff --git a/bin/log-performance-results.js b/bin/log-performance-results.js
index f18e40fea3d2f4..da22253546eb61 100755
--- a/bin/log-performance-results.js
+++ b/bin/log-performance-results.js
@@ -50,7 +50,7 @@ const data = new TextEncoder().encode(
 						performanceResults[ index ][ hash ] ?? {}
 					).map( ( [ key, value ] ) => [
 						metricsPrefix + key,
-						value,
+						typeof value === 'object' ? value.q50 : value,
 					] )
 				),
 			};
@@ -64,7 +64,7 @@ const data = new TextEncoder().encode(
 							performanceResults[ index ][ baseHash ] ?? {}
 						).map( ( [ key, value ] ) => [
 							metricsPrefix + key,
-							value,
+							typeof value === 'object' ? value.q50 : value,
 						] )
 					),
 				};
diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js
index 65b8a770d3764a..5ab597616f78d8 100644
--- a/bin/plugin/commands/performance.js
+++ b/bin/plugin/commands/performance.js
@@ -20,7 +20,7 @@ const config = require( '../config' );
 
 const ARTIFACTS_PATH =
 	process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
-const RESULTS_FILE_SUFFIX = '.performance-results.json';
+const RESULTS_FILE_SUFFIX = '.performance-results.raw.json';
 
 /**
  * @typedef WPPerformanceCommandOptions
@@ -56,24 +56,97 @@ function sanitizeBranchName( branch ) {
 }
 
 /**
- * Computes the median number from an array numbers.
- *
+ * @param {number} number
+ */
+function fixed( number ) {
+	return Math.round( number * 100 ) / 100;
+}
+
+/**
  * @param {number[]} array
- *
- * @return {number|undefined} Median value or undefined if array empty.
  */
-function median( array ) {
-	if ( ! array || ! array.length ) {
-		return undefined;
+function quartiles( array ) {
+	const numbers = array.slice().sort( ( a, b ) => a - b );
+
+	/**
+	 * @param {number} offset
+	 * @param {number} length
+	 */
+	function med( offset, length ) {
+		if ( length % 2 === 0 ) {
+			// even length, average of two middle numbers
+			return (
+				( numbers[ offset + length / 2 - 1 ] +
+					numbers[ offset + length / 2 ] ) /
+				2
+			);
+		}
+
+		// odd length, exact middle point
+		return numbers[ offset + ( length - 1 ) / 2 ];
 	}
 
-	const numbers = [ ...array ].sort( ( a, b ) => a - b );
-	const middleIndex = Math.floor( numbers.length / 2 );
+	const q50 = med( 0, numbers.length );
 
+	let q25, q75;
 	if ( numbers.length % 2 === 0 ) {
-		return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
+		// medians of two exact halves
+		const mid = numbers.length / 2;
+		q25 = med( 0, mid );
+		q75 = med( mid, mid );
+	} else {
+		// quartiles are average of medians of the smaller and bigger slice
+		const midl = ( numbers.length - 1 ) / 2;
+		const midh = ( numbers.length + 1 ) / 2;
+		q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2;
+		q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2;
 	}
-	return numbers[ middleIndex ];
+	return { q25, q50, q75 };
+}
+
+/**
+ * @param {number[]|undefined} values
+ */
+function stats( values ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const cnt = values.length;
+	return {
+		q25: fixed( q25 ),
+		q50: fixed( q50 ),
+		q75: fixed( q75 ),
+		cnt,
+	};
+}
+
+/**
+ * Nicely formats a given value.
+ *
+ * @param {string} metric Metric.
+ * @param {number} value
+ */
+function formatValue( metric, value ) {
+	if ( 'wpMemoryUsage' === metric ) {
+		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
+	}
+
+	if ( 'wpDbQueries' === metric ) {
+		return value.toString();
+	}
+
+	return `${ value } ms`;
+}
+
+/**
+ * @param {string}                 m
+ * @param {Record<string, number>} s
+ */
+function printStats( m, s ) {
+	const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 );
+	const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 );
+	return `${ formatValue( m, s.q50 ) } ms (±${ pp }/${ mp }%)`;
 }
 
 /**
@@ -151,24 +224,6 @@ function formatAsMarkdownTable( rows ) {
 	return result;
 }
 
-/**
- * Nicely formats a given value.
- *
- * @param {string} metric Metric.
- * @param {number} value
- */
-function formatValue( metric, value ) {
-	if ( 'wpMemoryUsage' === metric ) {
-		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
-	}
-
-	if ( 'wpDbQueries' === metric ) {
-		return value.toString();
-	}
-
-	return `${ value } ms`;
-}
-
 /**
  * Runs the performances tests on an array of branches and output the result.
  *
@@ -439,7 +494,7 @@ async function runPerformanceTests( branches, options ) {
 	const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
 		file.endsWith( RESULTS_FILE_SUFFIX )
 	);
-	/** @type {Record<string,Record<string, Record<string, number>>>} */
+	/** @type {Record<string,Record<string, Record<string, Record<string, number>>>>} */
 	const results = {};
 
 	// Calculate medians from all rounds.
@@ -464,11 +519,11 @@ async function runPerformanceTests( branches, options ) {
 			results[ testSuite ][ branch ] = {};
 
 			for ( const metric of metrics ) {
-				const values = resultsRounds
-					.map( ( round ) => round[ metric ] )
-					.filter( ( value ) => typeof value === 'number' );
+				const values = resultsRounds.flatMap(
+					( round ) => round[ metric ] ?? []
+				);
 
-				const value = median( values );
+				const value = stats( values );
 				if ( value !== undefined ) {
 					results[ testSuite ][ branch ][ metric ] = value;
 				}
@@ -506,45 +561,50 @@ async function runPerformanceTests( branches, options ) {
 		logAtIndent( 0, formats.success( testSuite ) );
 
 		// Invert the results so we can display them in a table.
-		/** @type {Record<string, Record<string, string>>} */
+		/** @type {Record<string, Record<string, Record<string, number>>>} */
 		const invertedResult = {};
 		for ( const [ branch, metrics ] of Object.entries(
 			results[ testSuite ]
 		) ) {
 			for ( const [ metric, value ] of Object.entries( metrics ) ) {
 				invertedResult[ metric ] = invertedResult[ metric ] || {};
-				invertedResult[ metric ][ branch ] = formatValue(
-					metric,
-					value
-				);
+				invertedResult[ metric ][ branch ] = value;
 			}
 		}
 
-		if ( branches.length === 2 ) {
-			const [ branch1, branch2 ] = branches;
-			for ( const metric in invertedResult ) {
-				const value1 = parseFloat(
-					invertedResult[ metric ][ branch1 ]
+		/** @type {Record<string, Record<string, string>>} */
+		const printedResult = {};
+		for ( const [ metric, branch ] of Object.entries( invertedResult ) ) {
+			printedResult[ metric ] = {};
+			for ( const [ branchName, data ] of Object.entries( branch ) ) {
+				printedResult[ metric ][ branchName ] = printStats(
+					metric,
+					data
 				);
-				const value2 = parseFloat(
-					invertedResult[ metric ][ branch2 ]
+			}
+
+			if ( branches.length === 2 ) {
+				const [ branch1, branch2 ] = branches;
+				const value1 = branch[ branch1 ].q50;
+				const value2 = branch[ branch2 ].q50;
+				const percentageChange = fixed(
+					( ( value1 - value2 ) / value2 ) * 100
 				);
-				const percentageChange = ( ( value1 - value2 ) / value2 ) * 100;
-				invertedResult[ metric ][
+				printedResult[ metric ][
 					'% Change'
-				] = `${ percentageChange.toFixed( 2 ) }%`;
+				] = `${ percentageChange }%`;
 			}
 		}
 
 		// Print the results.
-		console.table( invertedResult );
+		console.table( printedResult );
 
 		// Use yet another structure to generate a Markdown table.
 
 		const rows = [];
 
 		for ( const [ metric, resultBranches ] of Object.entries(
-			invertedResult
+			printedResult
 		) ) {
 			/**
 			 * @type {Record< string, string >}
diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts
index 779013684a0736..a4a7b2506fb01a 100644
--- a/test/performance/config/performance-reporter.ts
+++ b/test/performance/config/performance-reporter.ts
@@ -13,7 +13,7 @@ import type {
 /**
  * Internal dependencies
  */
-import { quartiles, round } from '../utils';
+import { stats, round } from '../utils';
 
 export interface WPRawPerformanceResults {
 	timeToFirstByte: number[];
@@ -47,7 +47,6 @@ type PerformanceStats = {
 	q25: number;
 	q50: number;
 	q75: number;
-	out: number[]; // outliers
 	cnt: number; // number of data points
 };
 
@@ -79,25 +78,6 @@ export interface WPPerformanceResults {
 	wpDbQueries?: PerformanceStats;
 }
 
-function stats( values: number[] ): PerformanceStats | undefined {
-	if ( ! values || values.length === 0 ) {
-		return undefined;
-	}
-	const { q25, q50, q75 } = quartiles( values );
-	const iqr = q75 - q25;
-	const out = values.filter(
-		( v ) => v > q75 + 1.5 * iqr || v < q25 - 1.5 * iqr
-	);
-	const cnt = values.length;
-	return {
-		q25: round( q25 ),
-		q50: round( q50 ),
-		q75: round( q75 ),
-		out: out.map( ( n ) => round( n ) ),
-		cnt,
-	};
-}
-
 /**
  * Curate the raw performance results.
  *
@@ -174,21 +154,13 @@ class PerformanceReporter implements Reporter {
 
 			const curatedResults = curateResults( JSON.parse( resultsBody ) );
 
-			// For now, to keep back compat, save only the medians, not the full stats.
-			const savedResults = Object.fromEntries(
-				Object.entries( curatedResults ).map( ( [ key, value ] ) => [
-					key,
-					value.q50,
-				] )
-			);
-
 			// Save curated results to file.
 			writeFileSync(
 				path.join(
 					resultsPath,
 					`${ resultsId }.performance-results.json`
 				),
-				JSON.stringify( savedResults, null, 2 )
+				JSON.stringify( curatedResults, null, 2 )
 			);
 
 			this.results[ testSuite ] = curatedResults;
@@ -209,18 +181,12 @@ class PerformanceReporter implements Reporter {
 			const printableResults: Record< string, { value: string } > = {};
 
 			for ( const [ key, value ] of Object.entries( results ) ) {
-				const p = value.q75 - value.q50;
-				const pp = round( ( 100 * p ) / value.q50 );
-				const m = value.q50 - value.q25;
-				const mp = round( ( 100 * m ) / value.q50 );
-				const outs =
-					value.out.length > 0
-						? ' [' + value.out.join( ', ' ) + ']'
-						: '';
+				const p = round( value.q75 - value.q50 );
+				const pp = round( 100 * ( p / value.q50 ) );
+				const m = round( value.q50 - value.q25 );
+				const mp = round( 100 * ( m / value.q50 ) );
 				printableResults[ key ] = {
-					value: `${ value.q50 } ±${ round( p ) }/${ round(
-						m
-					) } ms (±${ pp }/${ mp }%)${ outs } (${ value.cnt })`,
+					value: `${ value.q50 } ±${ p }/${ m } ms (±${ pp }/${ mp }%) (${ value.cnt })`,
 				};
 			}
 
diff --git a/test/performance/utils.js b/test/performance/utils.js
index 2128b49d9e7925..b30df24b130f6a 100644
--- a/test/performance/utils.js
+++ b/test/performance/utils.js
@@ -79,6 +79,20 @@ export function quartiles( array ) {
 	return { q25, q50, q75 };
 }
 
+export function stats( values ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const cnt = values.length;
+	return {
+		q25: round( q25 ),
+		q50: round( q50 ),
+		q75: round( q75 ),
+		cnt,
+	};
+}
+
 export function minimum( array ) {
 	if ( ! array || ! array.length ) {
 		return undefined;

From cd450aec6e9691281a9a880ba72f18bc48b3687a Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Sat, 11 May 2024 08:57:12 +0200
Subject: [PATCH 5/6] Format stats better

---
 bin/plugin/commands/performance.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js
index 5ab597616f78d8..428e82c2f12127 100644
--- a/bin/plugin/commands/performance.js
+++ b/bin/plugin/commands/performance.js
@@ -146,7 +146,7 @@ function formatValue( metric, value ) {
 function printStats( m, s ) {
 	const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 );
 	const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 );
-	return `${ formatValue( m, s.q50 ) } ms (±${ pp }/${ mp }%)`;
+	return `${ formatValue( m, s.q50 ) } +${ pp }% -${ mp }% (${ s.cnt })`;
 }
 
 /**

From f1e32b6627e979fabc0c5a32be6727ed4447fd5e Mon Sep 17 00:00:00 2001
From: Jarda Snajdr <jsnajdr@gmail.com>
Date: Wed, 15 May 2024 10:42:46 +0200
Subject: [PATCH 6/6] Remove counts, unify formatting in reporter vs CI

---
 bin/plugin/commands/performance.js            |  2 +-
 .../config/performance-reporter.ts            | 30 ++++++++++++++-----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js
index 428e82c2f12127..4c650bb4c7d138 100644
--- a/bin/plugin/commands/performance.js
+++ b/bin/plugin/commands/performance.js
@@ -146,7 +146,7 @@ function formatValue( metric, value ) {
 function printStats( m, s ) {
 	const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 );
 	const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 );
-	return `${ formatValue( m, s.q50 ) } +${ pp }% -${ mp }% (${ s.cnt })`;
+	return `${ formatValue( m, s.q50 ) } +${ pp }% -${ mp }%`;
 }
 
 /**
diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts
index a4a7b2506fb01a..049a21fb333b18 100644
--- a/test/performance/config/performance-reporter.ts
+++ b/test/performance/config/performance-reporter.ts
@@ -121,6 +121,19 @@ export function curateResults(
 			.filter( ( [ _, value ] ) => value !== undefined )
 	);
 }
+
+function formatValue( metric: string, value: number ) {
+	if ( 'wpMemoryUsage' === metric ) {
+		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
+	}
+
+	if ( 'wpDbQueries' === metric ) {
+		return value.toString();
+	}
+
+	return `${ value } ms`;
+}
+
 class PerformanceReporter implements Reporter {
 	private results: Record< string, WPPerformanceResults >;
 
@@ -180,13 +193,16 @@ class PerformanceReporter implements Reporter {
 		for ( const [ testSuite, results ] of Object.entries( this.results ) ) {
 			const printableResults: Record< string, { value: string } > = {};
 
-			for ( const [ key, value ] of Object.entries( results ) ) {
-				const p = round( value.q75 - value.q50 );
-				const pp = round( 100 * ( p / value.q50 ) );
-				const m = round( value.q50 - value.q25 );
-				const mp = round( 100 * ( m / value.q50 ) );
-				printableResults[ key ] = {
-					value: `${ value.q50 } ±${ p }/${ m } ms (±${ pp }/${ mp }%) (${ value.cnt })`,
+			for ( const [ metric, value ] of Object.entries( results ) ) {
+				const valueStr = formatValue( metric, value.q50 );
+				const pp = round(
+					( 100 * ( value.q75 - value.q50 ) ) / value.q50
+				);
+				const mp = round(
+					( 100 * ( value.q50 - value.q25 ) ) / value.q50
+				);
+				printableResults[ metric ] = {
+					value: `${ valueStr } +${ pp }% -${ mp }%`,
 				};
 			}