From e4aa308ba55704cce835086a662ec915d782ef17 Mon Sep 17 00:00:00 2001 From: Jarda Snajdr Date: Fri, 10 May 2024 10:43:53 +0200 Subject: [PATCH] Calculate quartiles and outliers, report in table --- .../config/performance-reporter.ts | 116 ++++++++++-------- test/performance/utils.js | 39 +++++- 2 files changed, 102 insertions(+), 53 deletions(-) diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts index 3979966e35e2f..178c5e8429f58 100644 --- a/test/performance/config/performance-reporter.ts +++ b/test/performance/config/performance-reporter.ts @@ -13,7 +13,7 @@ import type { /** * Internal dependencies */ -import { average, variance, round } from '../utils'; +import { quartiles, round } from '../utils'; export interface WPRawPerformanceResults { timeToFirstByte: number[]; @@ -81,6 +81,25 @@ export interface WPPerformanceResults { navigateV?: number; } +function stats( values: number[] ) { + if ( ! values || values.length === 0 ) { + return undefined; + } + const { q25, q50, q75 } = quartiles( values ); + const iqr = q75 - q25; + const out = values.filter( + ( v ) => v > q75 + 1.5 * iqr || v < q25 - 1.5 * iqr + ); + const cnt = values.length; + return { + q25: round( q25 ), + q50: round( q50 ), + q75: round( q75 ), + out: out.map( ( n ) => round( n ) ), + cnt, + }; +} + /** * Curate the raw performance results. * @@ -92,59 +111,32 @@ export function curateResults( results: WPRawPerformanceResults ): WPPerformanceResults { const output = { - timeToFirstByte: average( results.timeToFirstByte ), - timeToFirstByteV: variance( results.timeToFirstByte ), - largestContentfulPaint: average( results.largestContentfulPaint ), - largestContentfulPaintV: variance( results.largestContentfulPaint ), - lcpMinusTtfb: average( results.lcpMinusTtfb ), - lcpMinusTtfbV: variance( results.lcpMinusTtfb ), - serverResponse: average( results.serverResponse ), - serverResponseV: variance( results.serverResponse ), - firstPaint: average( results.firstPaint ), - firstPaintV: variance( results.firstPaint ), - domContentLoaded: average( results.domContentLoaded ), - domContentLoadedV: variance( results.domContentLoaded ), - loaded: average( results.loaded ), - loadedV: variance( results.loaded ), - firstContentfulPaint: average( results.firstContentfulPaint ), - firstContentfulPaintV: variance( results.firstContentfulPaint ), - firstBlock: average( results.firstBlock ), - firstBlockV: variance( results.firstBlock ), - type: average( results.type ), - typeV: variance( results.type ), - typeWithoutInspector: average( results.typeWithoutInspector ), - typeWithoutInspectorV: variance( results.typeWithoutInspector ), - typeWithTopToolbar: average( results.typeWithTopToolbar ), - typeWithTopToolbarV: variance( results.typeWithTopToolbar ), - typeContainer: average( results.typeContainer ), - typeContainerV: variance( results.typeContainer ), - focus: average( results.focus ), - focusV: variance( results.focus ), - inserterOpen: average( results.inserterOpen ), - inserterOpenV: variance( results.inserterOpen ), - inserterSearch: average( results.inserterSearch ), - inserterSearchV: variance( results.inserterSearch ), - inserterHover: average( results.inserterHover ), - inserterHoverV: variance( results.inserterHover ), - loadPatterns: average( results.loadPatterns ), - loadPatternsV: variance( results.loadPatterns ), - listViewOpen: average( results.listViewOpen ), - listViewOpenV: variance( results.listViewOpen ), - navigate: average( results.navigate ), - navigateV: variance( results.navigate ), + timeToFirstByte: stats( results.timeToFirstByte ), + largestContentfulPaint: stats( results.largestContentfulPaint ), + lcpMinusTtfb: stats( results.lcpMinusTtfb ), + serverResponse: stats( results.serverResponse ), + firstPaint: stats( results.firstPaint ), + domContentLoaded: stats( results.domContentLoaded ), + loaded: stats( results.loaded ), + firstContentfulPaint: stats( results.firstContentfulPaint ), + firstBlock: stats( results.firstBlock ), + type: stats( results.type ), + typeWithoutInspector: stats( results.typeWithoutInspector ), + typeWithTopToolbar: stats( results.typeWithTopToolbar ), + typeContainer: stats( results.typeContainer ), + focus: stats( results.focus ), + inserterOpen: stats( results.inserterOpen ), + inserterSearch: stats( results.inserterSearch ), + inserterHover: stats( results.inserterHover ), + loadPatterns: stats( results.loadPatterns ), + listViewOpen: stats( results.listViewOpen ), + navigate: stats( results.navigate ), }; - return ( + return Object.fromEntries( Object.entries( output ) // Reduce the output to contain taken metrics only. - .filter( ( [ _, value ] ) => typeof value === 'number' ) - .reduce( - ( acc, [ key, value ] ) => ( { - ...acc, - [ key ]: round( value ), - } ), - {} - ) + .filter( ( [ _, value ] ) => value !== undefined ) ); } class PerformanceReporter implements Reporter { @@ -180,13 +172,21 @@ class PerformanceReporter implements Reporter { const curatedResults = curateResults( JSON.parse( resultsBody ) ); + // For now, to keep back compat, save only the medians, not the full stats. + const savedResults = Object.fromEntries( + Object.entries( curatedResults ).map( ( [ key, value ] ) => [ + key, + value.q50, + ] ) + ); + // Save curated results to file. writeFileSync( path.join( resultsPath, `${ resultsId }.performance-results.json` ), - JSON.stringify( curatedResults, null, 2 ) + JSON.stringify( savedResults, null, 2 ) ); this.results[ testSuite ] = curatedResults; @@ -207,7 +207,19 @@ class PerformanceReporter implements Reporter { const printableResults: Record< string, { value: string } > = {}; for ( const [ key, value ] of Object.entries( results ) ) { - printableResults[ key ] = { value: `${ value } ms` }; + const p = value.q75 - value.q50; + const pp = round( ( 100 * p ) / value.q50 ); + const m = value.q50 - value.q25; + const mp = round( ( 100 * m ) / value.q50 ); + const outs = + value.out.length > 0 + ? ' [' + value.out.join( ', ' ) + ']' + : ''; + printableResults[ key ] = { + value: `${ value.q50 } ±${ round( p ) }/${ round( + m + ) } ms (±${ pp }/${ mp }%)${ outs } (${ value.cnt })`, + }; } // eslint-disable-next-line no-console diff --git a/test/performance/utils.js b/test/performance/utils.js index 31dadccbeb328..2128b49d9e792 100644 --- a/test/performance/utils.js +++ b/test/performance/utils.js @@ -20,7 +20,9 @@ export function average( array ) { } export function variance( array ) { - if ( ! array || ! array.length ) return undefined; + if ( ! array || ! array.length ) { + return undefined; + } return Math.sqrt( sum( array.map( ( x ) => x ** 2 ) ) / array.length - @@ -42,6 +44,41 @@ export function median( array ) { return numbers[ middleIndex ]; } +export function quartiles( array ) { + const numbers = array.slice().sort( ( a, b ) => a - b ); + + function med( offset, length ) { + if ( length % 2 === 0 ) { + // even length, average of two middle numbers + return ( + ( numbers[ offset + length / 2 - 1 ] + + numbers[ offset + length / 2 ] ) / + 2 + ); + } + + // odd length, exact middle point + return numbers[ offset + ( length - 1 ) / 2 ]; + } + + const q50 = med( 0, numbers.length ); + + let q25, q75; + if ( numbers.length % 2 === 0 ) { + // medians of two exact halves + const mid = numbers.length / 2; + q25 = med( 0, mid ); + q75 = med( mid, mid ); + } else { + // quartiles are average of medians of the smaller and bigger slice + const midl = ( numbers.length - 1 ) / 2; + const midh = ( numbers.length + 1 ) / 2; + q25 = ( med( 0, midl ) + med( 0, midh ) ) / 2; + q75 = ( med( midl, midh ) + med( midh, midl ) ) / 2; + } + return { q25, q50, q75 }; +} + export function minimum( array ) { if ( ! array || ! array.length ) { return undefined;