diff --git a/bin/log-performance-results.js b/bin/log-performance-results.js index f18e40fea3d2f4..da22253546eb61 100755 --- a/bin/log-performance-results.js +++ b/bin/log-performance-results.js @@ -50,7 +50,7 @@ const data = new TextEncoder().encode( performanceResults[ index ][ hash ] ?? {} ).map( ( [ key, value ] ) => [ metricsPrefix + key, - value, + typeof value === 'object' ? value.q50 : value, ] ) ), }; @@ -64,7 +64,7 @@ const data = new TextEncoder().encode( performanceResults[ index ][ baseHash ] ?? {} ).map( ( [ key, value ] ) => [ metricsPrefix + key, - value, + typeof value === 'object' ? value.q50 : value, ] ) ), }; diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js index 65b8a770d3764a..4c650bb4c7d138 100644 --- a/bin/plugin/commands/performance.js +++ b/bin/plugin/commands/performance.js @@ -20,7 +20,7 @@ const config = require( '../config' ); const ARTIFACTS_PATH = process.env.WP_ARTIFACTS_PATH || path.join( process.cwd(), 'artifacts' ); -const RESULTS_FILE_SUFFIX = '.performance-results.json'; +const RESULTS_FILE_SUFFIX = '.performance-results.raw.json'; /** * @typedef WPPerformanceCommandOptions @@ -56,24 +56,97 @@ function sanitizeBranchName( branch ) { } /** - * Computes the median number from an array numbers. - * + * @param {number} number + */ +function fixed( number ) { + return Math.round( number * 100 ) / 100; +} + +/** * @param {number[]} array - * - * @return {number|undefined} Median value or undefined if array empty. */ -function median( array ) { - if ( ! array || ! array.length ) { - return undefined; +function quartiles( array ) { + const numbers = array.slice().sort( ( a, b ) => a - b ); + + /** + * @param {number} offset + * @param {number} length + */ + function med( offset, length ) { + if ( length % 2 === 0 ) { + // even length, average of two middle numbers + return ( + ( numbers[ offset + length / 2 - 1 ] + + numbers[ offset + length / 2 ] ) / + 2 + ); + } + + // odd length, exact middle point + return numbers[ offset + ( length - 1 ) / 2 ]; } - const numbers = [ ...array ].sort( ( a, b ) => a - b ); - const middleIndex = Math.floor( numbers.length / 2 ); + const q50 = med( 0, numbers.length ); + let q25, q75; if ( numbers.length % 2 === 0 ) { - return ( numbers[ middleIndex - 1 ] + numbers[ middleIndex ] ) / 2; + // medians of two exact halves + const mid = numbers.length / 2; + q25 = med( 0, mid ); + q75 = med( mid, mid ); + } else { + // quartiles are average of medians of the smaller and bigger slice + const midl = ( numbers.length - 1 ) / 2; + const midh = ( numbers.length + 1 ) / 2; + q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2; + q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2; } - return numbers[ middleIndex ]; + return { q25, q50, q75 }; +} + +/** + * @param {number[]|undefined} values + */ +function stats( values ) { + if ( ! values || values.length === 0 ) { + return undefined; + } + const { q25, q50, q75 } = quartiles( values ); + const cnt = values.length; + return { + q25: fixed( q25 ), + q50: fixed( q50 ), + q75: fixed( q75 ), + cnt, + }; +} + +/** + * Nicely formats a given value. + * + * @param {string} metric Metric. + * @param {number} value + */ +function formatValue( metric, value ) { + if ( 'wpMemoryUsage' === metric ) { + return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; + } + + if ( 'wpDbQueries' === metric ) { + return value.toString(); + } + + return `${ value } ms`; +} + +/** + * @param {string} m + * @param {Record} s + */ +function printStats( m, s ) { + const pp = fixed( ( 100 * ( s.q75 - s.q50 ) ) / s.q50 ); + const mp = fixed( ( 100 * ( s.q50 - s.q25 ) ) / s.q50 ); + return `${ formatValue( m, s.q50 ) } +${ pp }% -${ mp }%`; } /** @@ -151,24 +224,6 @@ function formatAsMarkdownTable( rows ) { return result; } -/** - * Nicely formats a given value. - * - * @param {string} metric Metric. - * @param {number} value - */ -function formatValue( metric, value ) { - if ( 'wpMemoryUsage' === metric ) { - return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; - } - - if ( 'wpDbQueries' === metric ) { - return value.toString(); - } - - return `${ value } ms`; -} - /** * Runs the performances tests on an array of branches and output the result. * @@ -439,7 +494,7 @@ async function runPerformanceTests( branches, options ) { const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) => file.endsWith( RESULTS_FILE_SUFFIX ) ); - /** @type {Record>>} */ + /** @type {Record>>>} */ const results = {}; // Calculate medians from all rounds. @@ -464,11 +519,11 @@ async function runPerformanceTests( branches, options ) { results[ testSuite ][ branch ] = {}; for ( const metric of metrics ) { - const values = resultsRounds - .map( ( round ) => round[ metric ] ) - .filter( ( value ) => typeof value === 'number' ); + const values = resultsRounds.flatMap( + ( round ) => round[ metric ] ?? [] + ); - const value = median( values ); + const value = stats( values ); if ( value !== undefined ) { results[ testSuite ][ branch ][ metric ] = value; } @@ -506,45 +561,50 @@ async function runPerformanceTests( branches, options ) { logAtIndent( 0, formats.success( testSuite ) ); // Invert the results so we can display them in a table. - /** @type {Record>} */ + /** @type {Record>>} */ const invertedResult = {}; for ( const [ branch, metrics ] of Object.entries( results[ testSuite ] ) ) { for ( const [ metric, value ] of Object.entries( metrics ) ) { invertedResult[ metric ] = invertedResult[ metric ] || {}; - invertedResult[ metric ][ branch ] = formatValue( - metric, - value - ); + invertedResult[ metric ][ branch ] = value; } } - if ( branches.length === 2 ) { - const [ branch1, branch2 ] = branches; - for ( const metric in invertedResult ) { - const value1 = parseFloat( - invertedResult[ metric ][ branch1 ] + /** @type {Record>} */ + const printedResult = {}; + for ( const [ metric, branch ] of Object.entries( invertedResult ) ) { + printedResult[ metric ] = {}; + for ( const [ branchName, data ] of Object.entries( branch ) ) { + printedResult[ metric ][ branchName ] = printStats( + metric, + data ); - const value2 = parseFloat( - invertedResult[ metric ][ branch2 ] + } + + if ( branches.length === 2 ) { + const [ branch1, branch2 ] = branches; + const value1 = branch[ branch1 ].q50; + const value2 = branch[ branch2 ].q50; + const percentageChange = fixed( + ( ( value1 - value2 ) / value2 ) * 100 ); - const percentageChange = ( ( value1 - value2 ) / value2 ) * 100; - invertedResult[ metric ][ + printedResult[ metric ][ '% Change' - ] = `${ percentageChange.toFixed( 2 ) }%`; + ] = `${ percentageChange }%`; } } // Print the results. - console.table( invertedResult ); + console.table( printedResult ); // Use yet another structure to generate a Markdown table. const rows = []; for ( const [ metric, resultBranches ] of Object.entries( - invertedResult + printedResult ) ) { /** * @type {Record< string, string >} diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts index b449e36540404e..049a21fb333b18 100644 --- a/test/performance/config/performance-reporter.ts +++ b/test/performance/config/performance-reporter.ts @@ -13,7 +13,7 @@ import type { /** * Internal dependencies */ -import { average, median, round } from '../utils'; +import { stats, round } from '../utils'; export interface WPRawPerformanceResults { timeToFirstByte: number[]; @@ -43,85 +43,97 @@ export interface WPRawPerformanceResults { wpDbQueries: number[]; } +type PerformanceStats = { + q25: number; + q50: number; + q75: number; + cnt: number; // number of data points +}; + export interface WPPerformanceResults { - timeToFirstByte?: number; - largestContentfulPaint?: number; - lcpMinusTtfb?: number; - serverResponse?: number; - firstPaint?: number; - domContentLoaded?: number; - loaded?: number; - firstContentfulPaint?: number; - firstBlock?: number; - type?: number; - typeWithoutInspector?: number; - typeWithTopToolbar?: number; - typeContainer?: number; - focus?: number; - inserterOpen?: number; - inserterSearch?: number; - inserterHover?: number; - loadPatterns?: number; - listViewOpen?: number; - navigate?: number; - wpBeforeTemplate?: number; - wpTemplate?: number; - wpTotal?: number; - wpMemoryUsage?: number; - wpDbQueries?: number; + timeToFirstByte?: PerformanceStats; + largestContentfulPaint?: PerformanceStats; + lcpMinusTtfb?: PerformanceStats; + serverResponse?: PerformanceStats; + firstPaint?: PerformanceStats; + domContentLoaded?: PerformanceStats; + loaded?: PerformanceStats; + firstContentfulPaint?: PerformanceStats; + firstBlock?: PerformanceStats; + type?: PerformanceStats; + typeWithoutInspector?: PerformanceStats; + typeWithTopToolbar?: PerformanceStats; + typeContainer?: PerformanceStats; + focus?: PerformanceStats; + inserterOpen?: PerformanceStats; + inserterSearch?: PerformanceStats; + inserterHover?: PerformanceStats; + loadPatterns?: PerformanceStats; + listViewOpen?: PerformanceStats; + navigate?: PerformanceStats; + wpBeforeTemplate?: PerformanceStats; + wpTemplate?: PerformanceStats; + wpTotal?: PerformanceStats; + wpMemoryUsage?: PerformanceStats; + wpDbQueries?: PerformanceStats; } /** * Curate the raw performance results. * - * @param {WPRawPerformanceResults} results - * - * @return {WPPerformanceResults} Curated Performance results. + * @param results Raw results. + * @return Curated statistics for the results. */ export function curateResults( results: WPRawPerformanceResults ): WPPerformanceResults { const output = { - timeToFirstByte: median( results.timeToFirstByte ), - largestContentfulPaint: median( results.largestContentfulPaint ), - lcpMinusTtfb: median( results.lcpMinusTtfb ), - serverResponse: median( results.serverResponse ), - firstPaint: median( results.firstPaint ), - domContentLoaded: median( results.domContentLoaded ), - loaded: median( results.loaded ), - firstContentfulPaint: median( results.firstContentfulPaint ), - firstBlock: median( results.firstBlock ), - type: average( results.type ), - typeWithoutInspector: average( results.typeWithoutInspector ), - typeWithTopToolbar: average( results.typeWithTopToolbar ), - typeContainer: average( results.typeContainer ), - focus: average( results.focus ), - inserterOpen: average( results.inserterOpen ), - inserterSearch: average( results.inserterSearch ), - inserterHover: average( results.inserterHover ), - loadPatterns: average( results.loadPatterns ), - listViewOpen: average( results.listViewOpen ), - navigate: median( results.navigate ), - wpBeforeTemplate: median( results.wpBeforeTemplate ), - wpTemplate: median( results.wpTemplate ), - wpTotal: median( results.wpTotal ), - wpMemoryUsage: median( results.wpMemoryUsage ), - wpDbQueries: median( results.wpDbQueries ), + timeToFirstByte: stats( results.timeToFirstByte ), + largestContentfulPaint: stats( results.largestContentfulPaint ), + lcpMinusTtfb: stats( results.lcpMinusTtfb ), + serverResponse: stats( results.serverResponse ), + firstPaint: stats( results.firstPaint ), + domContentLoaded: stats( results.domContentLoaded ), + loaded: stats( results.loaded ), + firstContentfulPaint: stats( results.firstContentfulPaint ), + firstBlock: stats( results.firstBlock ), + type: stats( results.type ), + typeWithoutInspector: stats( results.typeWithoutInspector ), + typeWithTopToolbar: stats( results.typeWithTopToolbar ), + typeContainer: stats( results.typeContainer ), + focus: stats( results.focus ), + inserterOpen: stats( results.inserterOpen ), + inserterSearch: stats( results.inserterSearch ), + inserterHover: stats( results.inserterHover ), + loadPatterns: stats( results.loadPatterns ), + listViewOpen: stats( results.listViewOpen ), + navigate: stats( results.navigate ), + wpBeforeTemplate: stats( results.wpBeforeTemplate ), + wpTemplate: stats( results.wpTemplate ), + wpTotal: stats( results.wpTotal ), + wpMemoryUsage: stats( results.wpMemoryUsage ), + wpDbQueries: stats( results.wpDbQueries ), }; - return ( + return Object.fromEntries( Object.entries( output ) // Reduce the output to contain taken metrics only. - .filter( ( [ _, value ] ) => typeof value === 'number' ) - .reduce( - ( acc, [ key, value ] ) => ( { - ...acc, - [ key ]: round( value ), - } ), - {} - ) + .filter( ( [ _, value ] ) => value !== undefined ) ); } + +function formatValue( metric: string, value: number ) { + if ( 'wpMemoryUsage' === metric ) { + return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; + } + + if ( 'wpDbQueries' === metric ) { + return value.toString(); + } + + return `${ value } ms`; +} + class PerformanceReporter implements Reporter { private results: Record< string, WPPerformanceResults >; @@ -181,8 +193,17 @@ class PerformanceReporter implements Reporter { for ( const [ testSuite, results ] of Object.entries( this.results ) ) { const printableResults: Record< string, { value: string } > = {}; - for ( const [ key, value ] of Object.entries( results ) ) { - printableResults[ key ] = { value: `${ value } ms` }; + for ( const [ metric, value ] of Object.entries( results ) ) { + const valueStr = formatValue( metric, value.q50 ); + const pp = round( + ( 100 * ( value.q75 - value.q50 ) ) / value.q50 + ); + const mp = round( + ( 100 * ( value.q50 - value.q25 ) ) / value.q50 + ); + printableResults[ metric ] = { + value: `${ valueStr } +${ pp }% -${ mp }%`, + }; } // eslint-disable-next-line no-console diff --git a/test/performance/utils.js b/test/performance/utils.js index 72a851a4ffabc6..b30df24b130f6a 100644 --- a/test/performance/utils.js +++ b/test/performance/utils.js @@ -19,6 +19,17 @@ export function average( array ) { return sum( array ) / array.length; } +export function variance( array ) { + if ( ! array || ! array.length ) { + return undefined; + } + + return Math.sqrt( + sum( array.map( ( x ) => x ** 2 ) ) / array.length - + ( sum( array ) / array.length ) ** 2 + ); +} + export function median( array ) { if ( ! array || ! array.length ) { return undefined; @@ -33,6 +44,55 @@ export function median( array ) { return numbers[ middleIndex ]; } +export function quartiles( array ) { + const numbers = array.slice().sort( ( a, b ) => a - b ); + + function med( offset, length ) { + if ( length % 2 === 0 ) { + // even length, average of two middle numbers + return ( + ( numbers[ offset + length / 2 - 1 ] + + numbers[ offset + length / 2 ] ) / + 2 + ); + } + + // odd length, exact middle point + return numbers[ offset + ( length - 1 ) / 2 ]; + } + + const q50 = med( 0, numbers.length ); + + let q25, q75; + if ( numbers.length % 2 === 0 ) { + // medians of two exact halves + const mid = numbers.length / 2; + q25 = med( 0, mid ); + q75 = med( mid, mid ); + } else { + // quartiles are average of medians of the smaller and bigger slice + const midl = ( numbers.length - 1 ) / 2; + const midh = ( numbers.length + 1 ) / 2; + q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2; + q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2; + } + return { q25, q50, q75 }; +} + +export function stats( values ) { + if ( ! values || values.length === 0 ) { + return undefined; + } + const { q25, q50, q75 } = quartiles( values ); + const cnt = values.length; + return { + q25: round( q25 ), + q50: round( q50 ), + q75: round( q75 ), + cnt, + }; +} + export function minimum( array ) { if ( ! array || ! array.length ) { return undefined;