[PERF] Move performance testing logic into python scripts (dotnet#101143

) * Initial debugging work for python pipeline * Use custom performance branch * Remove template reference * Move resources to perf.yml * Ensure runtime repo is checked out to SourcesDirectory * Try fix python run * Set python in perf-job.yml * More fixes * More fixes * Set most params in perf-job.yml * Fix variable declaration * Fix some bugs in setup parameters * Ensure Helix Access Token is passed in * Fix osGroup arg parsing * Fix ios args and use local commit time * Fix physical promotion yaml * Fix project file path * Ensure internal param is passed * Ensure project file is passed in * Try dynamically load performance repo * Try use condition inside repositories definition * Try put condition on repositories object * Try use variables to conditionally create repository * Fix conditional variable definition * Just use ADO for performance repo * Use updated repository name for scenarios jobs * Use HELIX_WORKITEM_ROOT for performance dir * Fix paths for android APKs * Remove accidental submodule * Use new project file location for arm64 jit job * Move archive of work item dir to python * Use github for performance repo checkout on public jobs * Another attempt at conditional repository definition * Another attempt at conditional repository * Use multi repo checkout only for internal runs * Ensure ios scenarios and wasm jobs are working * Only copy files from dotnet-runtime directory on internal jobs * Use branch instead of ref for performance repository clone * Clone performance repo to correct folder in public jobs
AndyAyersMS · Sep 16, 2024 · b14e2f5 · b14e2f5
1 parent 657f18d
commit b14e2f5
Show file tree

Hide file tree

Showing 19 changed files with 207 additions and 1,357 deletions.
diff --git a/eng/pipelines/common/perf-variables.yml b/eng/pipelines/common/perf-variables.yml
diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml
@@ -58,6 +58,4 @@ variables:
                 ne(variables['isExtraPlatformsBuild'], true),
                 eq(variables['isRollingBuild'], true))) ]
 
-- template: /eng/pipelines/common/perf-variables.yml
-
-- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml
+- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml
diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml
@@ -1,3 +1,6 @@
+parameters:
+  perfBranch: 'main'
+
 jobs:
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}:
@@ -100,6 +103,7 @@ jobs:
   - template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml
     parameters:
       hybridGlobalization: True
+      perfBranch: ${{ parameters.perfBranch }}
 
   # run android scenarios
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -112,10 +116,11 @@ jobs:
       jobParameters:
         testGroup: perf
         runtimeType: AndroidMono
-        projectFile: android_scenarios.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj
         runKind: android_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         logicalmachine: 'perfpixel4a'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -129,10 +134,11 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: mono
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono interpreter perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -147,10 +153,11 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: mono
         codeGenType: 'Interpreter'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono aot microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -165,10 +172,11 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: mono
         codeGenType: 'AOT'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -184,10 +192,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -200,11 +209,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         pgoRunType: -NoDynamicPGO
+        perfBranch: ${{ parameters.perfBranch }}
 
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
@@ -216,11 +226,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         pgoRunType: --nodynamicpgo
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks no R2R perf jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -233,11 +244,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         r2rRunType: -NoR2R
+        perfBranch: ${{ parameters.perfBranch }}
 
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
@@ -249,11 +261,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         r2rRunType: --nor2r
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfowl microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -267,10 +280,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfowl'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfviper microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -284,10 +298,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfviper'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -301,11 +316,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfviper'
         experimentName: 'jitoptrepeat'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr crossgen perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -319,10 +335,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: crossgen_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj
         runKind: crossgen_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         logicalmachine: 'perftiger_crossgen'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # build mono runtime packs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -368,3 +385,4 @@ jobs:
               archiveExtension: '.tar.gz'
               archiveType: tar
               tarCompression: gz
+              perfBranch: ${{ parameters.perfBranch }}
diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml
@@ -7,7 +7,7 @@ parameters:
   onlySanityCheck: false
   downloadSpecificBuild: null # { buildId, pipeline, branchName, project }
   collectHelixLogsScript: ''
-  perfForkToUse: {} # url, branch
+  perfBranch: 'main'
 
 jobs:
 
@@ -43,14 +43,14 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
         codeGenType: 'wasm'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         javascriptEngine: 'javascriptcore'
         collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
         #downloadSpecificBuild:
           #buildId: '1693181'
           #pipeline: 'perf-wasm'
@@ -69,14 +69,14 @@ jobs:
         livelibrariesbuildconfig: Release
         runtimetype: wasm
         codegentype: 'aot'
-        projectfile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runkind: micro
         runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         javascriptengine: 'javascriptcore'
         collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
 - ${{ if eq(parameters.runProfile, 'v8') }}:
   - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
@@ -110,7 +110,7 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
         codeGenType: 'wasm'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
@@ -121,7 +121,7 @@ jobs:
         compare: ${{ parameters.compare }}
         onlySanityCheck: ${{ parameters.onlySanityCheck }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
   #run mono wasm aot microbenchmarks perf job
   # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101
@@ -138,7 +138,7 @@ jobs:
           livelibrariesbuildconfig: Release
           runtimetype: wasm
           codegentype: 'aot'
-          projectfile: microbenchmarks.proj
+          projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
           runkind: micro
           runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
           logicalmachine: 'perftiger'
@@ -149,7 +149,7 @@ jobs:
           compare: ${{ parameters.compare }}
           onlySanityCheck: ${{ parameters.onlySanityCheck }}
           downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-          perfForkToUse: ${{ parameters.perfForkToUse }}
+          perfBranch: ${{ parameters.perfBranch }}
 
   # run mono wasm blazor perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -163,14 +163,14 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
-        projectFile: blazor_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
         runKind: blazor_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
-        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
+        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
         logicalmachine: 'perftiger'
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
 - ${{if or(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}:
   # run mono wasm blazor perf job
@@ -185,12 +185,12 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
-        projectFile: blazor_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
         runKind: blazor_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
-        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
+        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
         logicalmachine: 'perftiger'
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
         hybridGlobalization: True
+        perfBranch: ${{ parameters.perfBranch }}
diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml
@@ -1,3 +1,9 @@
+parameters:
+- name: perfBranch
+  displayName: Performance Repo Branch
+  type: string
+  default: 'main'
+
 trigger:
   batch: true
   branches:
@@ -19,7 +25,7 @@ trigger:
     - THIRD-PARTY-NOTICES.TXT
 
 variables:
-  - template: /eng/pipelines/common/variables.yml
+- template: /eng/pipelines/common/variables.yml
 
 #
 # For the 'schedule' case, only wasm/jsc perf jobs are run.
@@ -46,9 +52,12 @@ extends:
       - template: /eng/pipelines/coreclr/perf-wasm-jobs.yml
         parameters:
           collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }}
+          perfBranch: ${{ parameters.perfBranch }}
           #${{ and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}:
           #  runProfile: 'non-v8'
           ${{ if ne(variables['System.TeamProject'], 'public') }}:
             runProfile: 'v8'
 
       - template: /eng/pipelines/coreclr/perf-non-wasm-jobs.yml
+        parameters:
+          perfBranch: ${{ parameters.perfBranch }}