Calculate and report quartiles in performance results

bin/log-performance-results.js

@@ -50,7 +50,7 @@ const data = new TextEncoder().encode(
 						performanceResults[ index ][ hash ] ?? {}
 					).map( ( [ key, value ] ) => [
 						metricsPrefix + key,
-						value,
+						typeof value === 'object' ? value.q50 : value,
 					] )
 				),
 			};
@@ -64,7 +64,7 @@ const data = new TextEncoder().encode(
 							performanceResults[ index ][ baseHash ] ?? {}
 						).map( ( [ key, value ] ) => [
 							metricsPrefix + key,
-							value,
+							typeof value === 'object' ? value.q50 : value,
 						] )
 					),
 				};

bin/plugin/commands/performance.js

@@ -20,7 +20,7 @@ const config = require( '../config' );
 
 const ARTIFACTS_PATH =
 	process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' );
-const RESULTS_FILE_SUFFIX = '.performance-results.json';
+const RESULTS_FILE_SUFFIX = '.performance-results.raw.json';
 
 /**
  * @typedef WPPerformanceCommandOptions
@@ -56,24 +56,97 @@ function sanitizeBranchName( branch ) {
 }
 
 /**
- * Computes the median number from an array numbers.
- *
+ * @param {number} number
+ */
+function fixed( number ) {
+	return Math.round( number * 100 ) / 100;
+}
+
+/**
  * @param {number[]} array
- *
- * @return {number|undefined} Median value or undefined if array empty.
  */
-function median( array ) {
-	if ( ! array || ! array.length ) {
-		return undefined;
+function quartiles( array ) {
+	const numbers = array.slice().sort( ( a, b ) => a - b );
+
+	/**
+	 * @param {number} offset
+	 * @param {number} length
+	 */
+	function med( offset, length ) {
+		if ( length % 2 === 0 ) {
+			// even length, average of two middle numbers
+			return (
+				( numbers[ offset + length / 2 - 1 ] +
+					numbers[ offset + length / 2 ] ) /
+				2
+			);
+		}
+
+		// odd length, exact middle point
+		return numbers[ offset + ( length - 1 ) / 2 ];
 	}
 
-	const numbers = [ ...array ].sort( ( a, b ) => a - b );
-	const middleIndex = Math.floor( numbers.length / 2 );
+	const q50 = med( 0, numbers.length );
 
+	let q25, q75;
 	if ( numbers.length % 2 === 0 ) {
-		return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2;
+		// medians of two exact halves
+		const mid = numbers.length / 2;
+		q25 = med( 0, mid );
+		q75 = med( mid, mid );
+	} else {
+		// quartiles are average of medians of the smaller and bigger slice
+		const midl = ( numbers.length - 1 ) / 2;
+		const midh = ( numbers.length + 1 ) / 2;
+		q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2;
+		q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2;
 	}
-	return numbers[ middleIndex ];
+	return { q25, q50, q75 };
+}
+
+/**
+ * @param {number[]|undefined} values
+ */
+function stats( values ) {
+	if ( ! values || values.length === 0 ) {
+		return undefined;
+	}
+	const { q25, q50, q75 } = quartiles( values );
+	const cnt = values.length;
+	return {
+		q25: fixed( q25 ),
+		q50: fixed( q50 ),
+		q75: fixed( q75 ),
+		cnt,
+	};
+}
+
+/**
+ * Nicely formats a given value.
+ *
+ * @param {string} metric Metric.
+ * @param {number} value
+ */
+function formatValue( metric, value ) {
+	if ( 'wpMemoryUsage' === metric ) {
+		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
+	}
+
+	if ( 'wpDbQueries' === metric ) {
+		return value.toString();
+	}
+
+	return `${ value } ms`;
+}
+
+/**
+ * @param {string}                 m
+ * @param {Record<string, number>} s
+ */
+function printStats( m, s ) {
+	const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 );
+	const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 );
+	return `${ formatValue( m, s.q50 ) } +${ pp }% -${ mp }%`;
 }
 
 /**
@@ -151,24 +224,6 @@ function formatAsMarkdownTable( rows ) {
 	return result;
 }
 
-/**
- * Nicely formats a given value.
- *
- * @param {string} metric Metric.
- * @param {number} value
- */
-function formatValue( metric, value ) {
-	if ( 'wpMemoryUsage' === metric ) {
-		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
-	}
-
-	if ( 'wpDbQueries' === metric ) {
-		return value.toString();
-	}
-
-	return `${ value } ms`;
-}
-
 /**
  * Runs the performances tests on an array of branches and output the result.
  *
@@ -439,7 +494,7 @@ async function runPerformanceTests( branches, options ) {
 	const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) =>
 		file.endsWith( RESULTS_FILE_SUFFIX )
 	);
-	/** @type {Record<string,Record<string, Record<string, number>>>} */
+	/** @type {Record<string,Record<string, Record<string, Record<string, number>>>>} */
 	const results = {};
 
 	// Calculate medians from all rounds.
@@ -464,11 +519,11 @@ async function runPerformanceTests( branches, options ) {
 			results[ testSuite ][ branch ] = {};
 
 			for ( const metric of metrics ) {
-				const values = resultsRounds
-					.map( ( round ) => round[ metric ] )
-					.filter( ( value ) => typeof value === 'number' );
+				const values = resultsRounds.flatMap(
+					( round ) => round[ metric ] ?? []
+				);
 
-				const value = median( values );
+				const value = stats( values );
 				if ( value !== undefined ) {
 					results[ testSuite ][ branch ][ metric ] = value;
 				}
@@ -506,45 +561,50 @@ async function runPerformanceTests( branches, options ) {
 		logAtIndent( 0, formats.success( testSuite ) );
 
 		// Invert the results so we can display them in a table.
-		/** @type {Record<string, Record<string, string>>} */
+		/** @type {Record<string, Record<string, Record<string, number>>>} */
 		const invertedResult = {};
 		for ( const [ branch, metrics ] of Object.entries(
 			results[ testSuite ]
 		) ) {
 			for ( const [ metric, value ] of Object.entries( metrics ) ) {
 				invertedResult[ metric ] = invertedResult[ metric ] || {};
-				invertedResult[ metric ][ branch ] = formatValue(
-					metric,
-					value
-				);
+				invertedResult[ metric ][ branch ] = value;
 			}
 		}
 
-		if ( branches.length === 2 ) {
-			const [ branch1, branch2 ] = branches;
-			for ( const metric in invertedResult ) {
-				const value1 = parseFloat(
-					invertedResult[ metric ][ branch1 ]
+		/** @type {Record<string, Record<string, string>>} */
+		const printedResult = {};
+		for ( const [ metric, branch ] of Object.entries( invertedResult ) ) {
+			printedResult[ metric ] = {};
+			for ( const [ branchName, data ] of Object.entries( branch ) ) {
+				printedResult[ metric ][ branchName ] = printStats(
+					metric,
+					data
 				);
-				const value2 = parseFloat(
-					invertedResult[ metric ][ branch2 ]
+			}
+
+			if ( branches.length === 2 ) {
+				const [ branch1, branch2 ] = branches;
+				const value1 = branch[ branch1 ].q50;
+				const value2 = branch[ branch2 ].q50;
+				const percentageChange = fixed(
+					( ( value1 - value2 ) / value2 ) * 100
 				);
-				const percentageChange = ( ( value1 - value2 ) / value2 ) * 100;
-				invertedResult[ metric ][
+				printedResult[ metric ][
 					'% Change'
-				] = `${ percentageChange.toFixed( 2 ) }%`;
+				] = `${ percentageChange }%`;
 			}
 		}
 
 		// Print the results.
-		console.table( invertedResult );
+		console.table( printedResult );
 
 		// Use yet another structure to generate a Markdown table.
 
 		const rows = [];
 
 		for ( const [ metric, resultBranches ] of Object.entries(
-			invertedResult
+			printedResult
 		) ) {
 			/**
 			 * @type {Record< string, string >}