From 2f4f6dfe6ea4cf5ab7a402f8a10c21cc4d64cf38 Mon Sep 17 00:00:00 2001 From: Jarda Snajdr Date: Fri, 10 May 2024 17:07:09 +0200 Subject: [PATCH] Calculate stats in CI, too --- bin/log-performance-results.js | 4 +- bin/plugin/commands/performance.js | 121 +++++++++++++----- .../config/performance-reporter.ts | 48 +------ test/performance/utils.js | 14 ++ 4 files changed, 113 insertions(+), 74 deletions(-) diff --git a/bin/log-performance-results.js b/bin/log-performance-results.js index f18e40fea3d2f4..da22253546eb61 100755 --- a/bin/log-performance-results.js +++ b/bin/log-performance-results.js @@ -50,7 +50,7 @@ const data = new TextEncoder().encode( performanceResults[ index ][ hash ] ?? {} ).map( ( [ key, value ] ) => [ metricsPrefix + key, - value, + typeof value === 'object' ? value.q50 : value, ] ) ), }; @@ -64,7 +64,7 @@ const data = new TextEncoder().encode( performanceResults[ index ][ baseHash ] ?? {} ).map( ( [ key, value ] ) => [ metricsPrefix + key, - value, + typeof value === 'object' ? value.q50 : value, ] ) ), }; diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js index 9d9b39fce09845..9bda9650f28f97 100644 --- a/bin/plugin/commands/performance.js +++ b/bin/plugin/commands/performance.js @@ -20,7 +20,7 @@ const config = require( '../config' ); const ARTIFACTS_PATH = process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' ); -const RESULTS_FILE_SUFFIX = '.performance-results.json'; +const RESULTS_FILE_SUFFIX = '.performance-results.raw.json'; /** * @typedef WPPerformanceCommandOptions @@ -56,24 +56,78 @@ function sanitizeBranchName( branch ) { } /** - * Computes the median number from an array numbers. - * + * @param {number} number + */ +function fixed( number ) { + return Math.round( number * 100 ) / 100; +} + +/** * @param {number[]} array - * - * @return {number|undefined} Median value or undefined if array empty. */ -function median( array ) { - if ( ! array || ! array.length ) { - return undefined; +function quartiles( array ) { + const numbers = array.slice().sort( ( a, b ) => a - b ); + + /** + * @param {number} offset + * @param {number} length + */ + function med( offset, length ) { + if ( length % 2 === 0 ) { + // even length, average of two middle numbers + return ( + ( numbers[ offset + length / 2 - 1 ] + + numbers[ offset + length / 2 ] ) / + 2 + ); + } + + // odd length, exact middle point + return numbers[ offset + ( length - 1 ) / 2 ]; } - const numbers = [ ...array ].sort( ( a, b ) => a - b ); - const middleIndex = Math.floor( numbers.length / 2 ); + const q50 = med( 0, numbers.length ); + let q25, q75; if ( numbers.length % 2 === 0 ) { - return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2; + // medians of two exact halves + const mid = numbers.length / 2; + q25 = med( 0, mid ); + q75 = med( mid, mid ); + } else { + // quartiles are average of medians of the smaller and bigger slice + const midl = ( numbers.length - 1 ) / 2; + const midh = ( numbers.length + 1 ) / 2; + q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2; + q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2; } - return numbers[ middleIndex ]; + return { q25, q50, q75 }; +} + +/** + * @param {number[]|undefined} values + */ +function stats( values ) { + if ( ! values || values.length === 0 ) { + return undefined; + } + const { q25, q50, q75 } = quartiles( values ); + const cnt = values.length; + return { + q25: fixed( q25 ), + q50: fixed( q50 ), + q75: fixed( q75 ), + cnt, + }; +} + +/** + * @param {Record} s + */ +function printStats( s ) { + const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 ); + const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 ); + return `${ s.q50 } ms (±${ pp }/${ mp }%)`; } /** @@ -366,7 +420,7 @@ async function runPerformanceTests( branches, options ) { const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) => file.endsWith( RESULTS_FILE_SUFFIX ) ); - /** @type {Record>>} */ + /** @type {Record>>>} */ const results = {}; // Calculate medians from all rounds. @@ -391,11 +445,11 @@ async function runPerformanceTests( branches, options ) { results[ testSuite ][ branch ] = {}; for ( const metric of metrics ) { - const values = resultsRounds - .map( ( round ) => round[ metric ] ) - .filter( ( value ) => typeof value === 'number' ); + const values = resultsRounds.flatMap( + ( round ) => round[ metric ] ?? [] + ); - const value = median( values ); + const value = stats( values ); if ( value !== undefined ) { results[ testSuite ][ branch ][ metric ] = value; } @@ -428,35 +482,40 @@ async function runPerformanceTests( branches, options ) { logAtIndent( 0, formats.success( testSuite ) ); // Invert the results so we can display them in a table. - /** @type {Record>} */ + /** @type {Record>>} */ const invertedResult = {}; for ( const [ branch, metrics ] of Object.entries( results[ testSuite ] ) ) { for ( const [ metric, value ] of Object.entries( metrics ) ) { invertedResult[ metric ] = invertedResult[ metric ] || {}; - invertedResult[ metric ][ branch ] = `${ value } ms`; + invertedResult[ metric ][ branch ] = value; } } - if ( branches.length === 2 ) { - const [ branch1, branch2 ] = branches; - for ( const metric in invertedResult ) { - const value1 = parseFloat( - invertedResult[ metric ][ branch1 ] - ); - const value2 = parseFloat( - invertedResult[ metric ][ branch2 ] + /** @type {Record>} */ + const printedResult = {}; + for ( const [ metric, branch ] of Object.entries( invertedResult ) ) { + printedResult[ metric ] = {}; + for ( const [ branchName, data ] of Object.entries( branch ) ) { + printedResult[ metric ][ branchName ] = printStats( data ); + } + + if ( branches.length === 2 ) { + const [ branch1, branch2 ] = branches; + const value1 = branch[ branch1 ].q50; + const value2 = branch[ branch2 ].q50; + const percentageChange = fixed( + ( ( value1 - value2 ) / value2 ) * 100 ); - const percentageChange = ( ( value1 - value2 ) / value2 ) * 100; - invertedResult[ metric ][ + printedResult[ metric ][ '% Change' - ] = `${ percentageChange.toFixed( 2 ) }%`; + ] = `${ percentageChange }%`; } } // Print the results. - console.table( invertedResult ); + console.table( printedResult ); } } diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts index 2ee1ee7cd24f4b..240165d3d2dda9 100644 --- a/test/performance/config/performance-reporter.ts +++ b/test/performance/config/performance-reporter.ts @@ -13,7 +13,7 @@ import type { /** * Internal dependencies */ -import { quartiles, round } from '../utils'; +import { stats, round } from '../utils'; export interface WPRawPerformanceResults { timeToFirstByte: number[]; @@ -42,7 +42,6 @@ type PerformanceStats = { q25: number; q50: number; q75: number; - out: number[]; // outliers cnt: number; // number of data points }; @@ -69,25 +68,6 @@ export interface WPPerformanceResults { navigate?: PerformanceStats; } -function stats( values: number[] ): PerformanceStats | undefined { - if ( ! values || values.length === 0 ) { - return undefined; - } - const { q25, q50, q75 } = quartiles( values ); - const iqr = q75 - q25; - const out = values.filter( - ( v ) => v > q75 + 1.5 * iqr || v < q25 - 1.5 * iqr - ); - const cnt = values.length; - return { - q25: round( q25 ), - q50: round( q50 ), - q75: round( q75 ), - out: out.map( ( n ) => round( n ) ), - cnt, - }; -} - /** * Curate the raw performance results. * @@ -159,21 +139,13 @@ class PerformanceReporter implements Reporter { const curatedResults = curateResults( JSON.parse( resultsBody ) ); - // For now, to keep back compat, save only the medians, not the full stats. - const savedResults = Object.fromEntries( - Object.entries( curatedResults ).map( ( [ key, value ] ) => [ - key, - value.q50, - ] ) - ); - // Save curated results to file. writeFileSync( path.join( resultsPath, `${ resultsId }.performance-results.json` ), - JSON.stringify( savedResults, null, 2 ) + JSON.stringify( curatedResults, null, 2 ) ); this.results[ testSuite ] = curatedResults; @@ -194,18 +166,12 @@ class PerformanceReporter implements Reporter { const printableResults: Record< string, { value: string } > = {}; for ( const [ key, value ] of Object.entries( results ) ) { - const p = value.q75 - value.q50; - const pp = round( ( 100 * p ) / value.q50 ); - const m = value.q50 - value.q25; - const mp = round( ( 100 * m ) / value.q50 ); - const outs = - value.out.length > 0 - ? ' [' + value.out.join( ', ' ) + ']' - : ''; + const p = round( value.q75 - value.q50 ); + const pp = round( 100 * ( p / value.q50 ) ); + const m = round( value.q50 - value.q25 ); + const mp = round( 100 * ( m / value.q50 ) ); printableResults[ key ] = { - value: `${ value.q50 } ±${ round( p ) }/${ round( - m - ) } ms (±${ pp }/${ mp }%)${ outs } (${ value.cnt })`, + value: `${ value.q50 } ±${ p }/${ m } ms (±${ pp }/${ mp }%) (${ value.cnt })`, }; } diff --git a/test/performance/utils.js b/test/performance/utils.js index 2128b49d9e7925..b30df24b130f6a 100644 --- a/test/performance/utils.js +++ b/test/performance/utils.js @@ -79,6 +79,20 @@ export function quartiles( array ) { return { q25, q50, q75 }; } +export function stats( values ) { + if ( ! values || values.length === 0 ) { + return undefined; + } + const { q25, q50, q75 } = quartiles( values ); + const cnt = values.length; + return { + q25: round( q25 ), + q50: round( q50 ), + q75: round( q75 ), + cnt, + }; +} + export function minimum( array ) { if ( ! array || ! array.length ) { return undefined;