Calculate stats in CI, too

bin/log-performance-results.js

@@ -50,7 +50,7 @@ const data = new TextEncoder().encode(
 						performanceResults[ index ][ hash ] ?? {}
 					).map( ( [ key, value ] ) => [
 						metricsPrefix + key,
-						value,
+						typeof value === 'object' ? value.q50 : value,
 					] )
 				),
 			};
@@ -64,7 +64,7 @@ const data = new TextEncoder().encode(
 							performanceResults[ index ][ baseHash ] ?? {}
 						).map( ( [ key, value ] ) => [
 							metricsPrefix + key,
-							value,
+							typeof value === 'object' ? value.q50 : value,
 						] )
 					),
 				};

bin/plugin/commands/performance.js

@@ -20,7 +20,7 @@ const config = require( '../config' );
 
 const ARTIFACTS_PATH =
 	process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
-const RESULTS_FILE_SUFFIX = '.performance-results.json';
+const RESULTS_FILE_SUFFIX = '.performance-results.raw.json';
 
 /**
  * @typedef WPPerformanceCommandOptions
@@ -56,24 +56,78 @@ function sanitizeBranchName( branch ) {
 }
 
 /**
- * Computes the median number from an array numbers.
- *
+ * @param {number} number
+ */
+function fixed( number ) {
+	return Math.round( number * 100 ) / 100;
+}
+
+/**
  * @param {number[]} array
- *
- * @return {number|undefined} Median value or undefined if array empty.
  */
-function median( array ) {
-	if ( ! array || ! array.length ) {
-		return undefined;
+function quartiles( array ) {
+	const numbers = array.slice().sort( ( a, b ) => a - b );
+
+	/**
+	 * @param {number} offset
+	 * @param {number} length
+	 */
+	function med( offset, length ) {
+		if ( length % 2 === 0 ) {
+			// even length, average of two middle numbers
+			return (
+				( numbers[ offset + length / 2 - 1 ] +
+					numbers[ offset + length / 2 ] ) /
+				2
+			);
+		}
+
+		// odd length, exact middle point
+		return numbers[ offset + ( length - 1 ) / 2 ];
 	}
 
-	const numbers = [ ...array ].sort( ( a, b ) => a - b );
-	const middleIndex = Math.floor( numbers.length / 2 );
+	const q50 = med( 0, numbers.length );
 
+	let q25, q75;
 	if ( numbers.length % 2 === 0 ) {
-		return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
+		// medians of two exact halves
+		const mid = numbers.length / 2;
+		q25 = med( 0, mid );
+		q75 = med( mid, mid );
+	} else {
+		// quartiles are average of medians of the smaller and bigger slice
+		const midl = ( numbers.length - 1 ) / 2;
+		const midh = ( numbers.length + 1 ) / 2;
+		q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2;
+		q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2;
 	}
-	return numbers[ middleIndex ];
+	return { q25, q50, q75 };
+}
+
+/**
+ * @param {number[]|undefined} values
+ */
+function stats( values ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const cnt = values.length;
+	return {
+		q25: fixed( q25 ),
+		q50: fixed( q50 ),
+		q75: fixed( q75 ),
+		cnt,
+	};
+}
+
+/**
+ * @param {Record<string, number>} s
+ */
+function printStats( s ) {
+	const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 );
+	const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 );
+	return `${ s.q50 } ms (±${ pp }/${ mp }%)`;
 }
 
 /**
@@ -366,7 +420,7 @@ async function runPerformanceTests( branches, options ) {
 	const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
 		file.endsWith( RESULTS_FILE_SUFFIX )
 	);
-	/** @type {Record<string,Record<string, Record<string, number>>>} */
+	/** @type {Record<string,Record<string, Record<string, Record<string, number>>>>} */
 	const results = {};
 
 	// Calculate medians from all rounds.
@@ -391,11 +445,11 @@ async function runPerformanceTests( branches, options ) {
 			results[ testSuite ][ branch ] = {};
 
 			for ( const metric of metrics ) {
-				const values = resultsRounds
-					.map( ( round ) => round[ metric ] )
-					.filter( ( value ) => typeof value === 'number' );
+				const values = resultsRounds.flatMap(
+					( round ) => round[ metric ] ?? []
+				);
 
-				const value = median( values );
+				const value = stats( values );
 				if ( value !== undefined ) {
 					results[ testSuite ][ branch ][ metric ] = value;
 				}
@@ -428,35 +482,40 @@ async function runPerformanceTests( branches, options ) {
 		logAtIndent( 0, formats.success( testSuite ) );
 
 		// Invert the results so we can display them in a table.
-		/** @type {Record<string, Record<string, string>>} */
+		/** @type {Record<string, Record<string, Record<string, number>>>} */
 		const invertedResult = {};
 		for ( const [ branch, metrics ] of Object.entries(
 			results[ testSuite ]
 		) ) {
 			for ( const [ metric, value ] of Object.entries( metrics ) ) {
 				invertedResult[ metric ] = invertedResult[ metric ] || {};
-				invertedResult[ metric ][ branch ] = `${ value } ms`;
+				invertedResult[ metric ][ branch ] = value;
 			}
 		}
 
-		if ( branches.length === 2 ) {
-			const [ branch1, branch2 ] = branches;
-			for ( const metric in invertedResult ) {
-				const value1 = parseFloat(
-					invertedResult[ metric ][ branch1 ]
-				);
-				const value2 = parseFloat(
-					invertedResult[ metric ][ branch2 ]
+		/** @type {Record<string, Record<string, string>>} */
+		const printedResult = {};
+		for ( const [ metric, branch ] of Object.entries( invertedResult ) ) {
+			printedResult[ metric ] = {};
+			for ( const [ branchName, data ] of Object.entries( branch ) ) {
+				printedResult[ metric ][ branchName ] = printStats( data );
+			}
+
+			if ( branches.length === 2 ) {
+				const [ branch1, branch2 ] = branches;
+				const value1 = branch[ branch1 ].q50;
+				const value2 = branch[ branch2 ].q50;
+				const percentageChange = fixed(
+					( ( value1 - value2 ) / value2 ) * 100
 				);
-				const percentageChange = ( ( value1 - value2 ) / value2 ) * 100;
-				invertedResult[ metric ][
+				printedResult[ metric ][
 					'% Change'
-				] = `${ percentageChange.toFixed( 2 ) }%`;
+				] = `${ percentageChange }%`;
 			}
 		}
 
 		// Print the results.
-		console.table( invertedResult );
+		console.table( printedResult );
 	}
 }
 

test/performance/config/performance-reporter.ts

@@ -13,7 +13,7 @@ import type {
 /**
  * Internal dependencies
  */
-import { quartiles, round } from '../utils';
+import { stats, round } from '../utils';
 
 export interface WPRawPerformanceResults {
 	timeToFirstByte: number[];
@@ -42,7 +42,6 @@ type PerformanceStats = {
 	q25: number;
 	q50: number;
 	q75: number;
-	out: number[]; // outliers
 	cnt: number; // number of data points
 };
 
@@ -69,25 +68,6 @@ export interface WPPerformanceResults {
 	navigate?: PerformanceStats;
 }
 
-function stats( values: number[] ): PerformanceStats | undefined {
-	if ( ! values || values.length === 0 ) {
-		return undefined;
-	}
-	const { q25, q50, q75 } = quartiles( values );
-	const iqr = q75 - q25;
-	const out = values.filter(
-		( v ) => v > q75 + 1.5 * iqr || v < q25 - 1.5 * iqr
-	);
-	const cnt = values.length;
-	return {
-		q25: round( q25 ),
-		q50: round( q50 ),
-		q75: round( q75 ),
-		out: out.map( ( n ) => round( n ) ),
-		cnt,
-	};
-}
-
 /**
  * Curate the raw performance results.
  *
@@ -159,21 +139,13 @@ class PerformanceReporter implements Reporter {
 
 			const curatedResults = curateResults( JSON.parse( resultsBody ) );
 
-			// For now, to keep back compat, save only the medians, not the full stats.
-			const savedResults = Object.fromEntries(
-				Object.entries( curatedResults ).map( ( [ key, value ] ) => [
-					key,
-					value.q50,
-				] )
-			);
-
 			// Save curated results to file.
 			writeFileSync(
 				path.join(
 					resultsPath,
 					`${ resultsId }.performance-results.json`
 				),
-				JSON.stringify( savedResults, null, 2 )
+				JSON.stringify( curatedResults, null, 2 )
 			);
 
 			this.results[ testSuite ] = curatedResults;
@@ -194,18 +166,12 @@ class PerformanceReporter implements Reporter {
 			const printableResults: Record< string, { value: string } > = {};
 
 			for ( const [ key, value ] of Object.entries( results ) ) {
-				const p = value.q75 - value.q50;
-				const pp = round( ( 100 * p ) / value.q50 );
-				const m = value.q50 - value.q25;
-				const mp = round( ( 100 * m ) / value.q50 );
-				const outs =
-					value.out.length > 0
-						? ' [' + value.out.join( ', ' ) + ']'
-						: '';
+				const p = round( value.q75 - value.q50 );
+				const pp = round( 100 * ( p / value.q50 ) );
+				const m = round( value.q50 - value.q25 );
+				const mp = round( 100 * ( m / value.q50 ) );
 				printableResults[ key ] = {
-					value: `${ value.q50 } ±${ round( p ) }/${ round(
-						m
-					) } ms (±${ pp }/${ mp }%)${ outs } (${ value.cnt })`,
+					value: `${ value.q50 } ±${ p }/${ m } ms (±${ pp }/${ mp }%) (${ value.cnt })`,
 				};
 			}
 

test/performance/utils.js

@@ -79,6 +79,20 @@ export function quartiles( array ) {
 	return { q25, q50, q75 };
 }
 
+export function stats( values ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const cnt = values.length;
+	return {
+		q25: round( q25 ),
+		q50: round( q50 ),
+		q75: round( q75 ),
+		cnt,
+	};
+}
+
 export function minimum( array ) {
 	if ( ! array || ! array.length ) {
 		return undefined;