From 6720879f73168e96d8d665a056dce3ed675800da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= Date: Tue, 1 Feb 2022 14:22:25 +0100 Subject: [PATCH 01/10] First attempt to add AD to the benchmarks --- benchmark/benchmarks.jl | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 0cb80e5ce..f49b75e51 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,5 +1,7 @@ using BenchmarkTools using KernelFunctions +using Zygote +using ForwardDiff N1 = 10 N2 = 20 @@ -17,24 +19,52 @@ Yv = collect.(eachcol(Y)) SUITE = BenchmarkGroup() kernels = Dict( - "SqExponential" => SqExponentialKernel(), "Exponential" => ExponentialKernel() + # Constant Kernels + "Constant" => ((2.0,), x->ConstantKernel(x)), + "White" => ((), ()->WhiteKernel()), + # Cosine Kernel + "Cosine" => ((), ()->CosineKernel()), + # Exponential Kernels + "Exponential" => ((), ()->ExponentialKernel()), + "Gibbs" => ((), ()->GibbsKernel(;lengthscale=sin)), + "SqExponential" => ((), ()->SqExponentialKernel()), + "GammaExponential" => ((1.0,), x->GammaExponentialKernel(;γ=2 * logistic(x))), + # Exponentiated Kernel + "Exponentiated" => ((), ()->ExponentiatedKernel()), ) inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv)) functions = Dict( - "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X), - "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y), - "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X), - "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y), + "kernelmatrixX" => (fk, args, X, Y) -> kernelmatrix(fk(args...), X), + "kernelmatrixXY" => (fk, args, X, Y) -> kernelmatrix(fk(args...), X, Y), + "kernelmatrix_diagX" => (fk, args, X, Y) -> kernelmatrix_diag(fk(args...), X), + "kernelmatrix_diagXY" => (fk, args, X, Y) -> kernelmatrix_diag(fk(args...), X, Y), ) -for (kname, kernel) in kernels - SUITE[kname] = sk = BenchmarkGroup() +# Test the allocated functions +SUITE["alloc_suite"] = s_alloc = BenchmarkGroup() +for (kname, (kargs, kf)) in kernels + s_alloc[kname] = sk = BenchmarkGroup() for (inputname, (X, Y)) in inputtypes sk[inputname] = si = BenchmarkGroup() for (fname, f) in functions - si[fname] = @benchmarkable $f($kernel, $X, $Y) + si[fname] = @benchmarkable $f($kf, $kargs, $X, $Y) + end + end +end + +# Test the AD frameworks +## Zygote +SUITE["zygote"] = s_zygote = BenchmarkGroup() +for (kname, (kargs, kf)) in kernels + s_zygote[kname] = sk = BenchmarkGroup() + for (inputname, (X, Y)) in inputtypes + sk[inputname] = si = BenchmarkGroup() + for (fname, f) in functions + si[fname] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, Y + $f($kf(args...), x, y) + end end end end From 7853609322bf99499368760e270508c99134a79b Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 15:23:05 +0100 Subject: [PATCH 02/10] Add more structure --- benchmark/Project.toml | 8 ++++++++ benchmark/benchmarks.jl | 42 +++++++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index cf9c7b9c1..aaa7b4a0c 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,3 +1,11 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392" +LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[compat] +BenchmarkTools = "1" +ForwardDiff = "0.10" +Zygote = "0.6" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index f49b75e51..92591d261 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,5 +1,7 @@ using BenchmarkTools using KernelFunctions +using ParameterHandling +using LogExpFunctions: logistic using Zygote using ForwardDiff @@ -18,9 +20,10 @@ Yv = collect.(eachcol(Y)) # Create the general suite of benchmarks SUITE = BenchmarkGroup() +# Create a list of kernel and their constructors kernels = Dict( # Constant Kernels - "Constant" => ((2.0,), x->ConstantKernel(x)), + "Constant" => ((2.0,), x->ConstantKernel(;c=x)), "White" => ((), ()->WhiteKernel()), # Cosine Kernel "Cosine" => ((), ()->CosineKernel()), @@ -43,34 +46,49 @@ functions = Dict( ) # Test the allocated functions -SUITE["alloc_suite"] = s_alloc = BenchmarkGroup() +SUITE["Allocated Functions"] = suite_alloc = BenchmarkGroup() for (kname, (kargs, kf)) in kernels - s_alloc[kname] = sk = BenchmarkGroup() + suite_alloc[kname] = suite_kernel = BenchmarkGroup() for (inputname, (X, Y)) in inputtypes - sk[inputname] = si = BenchmarkGroup() + suite_kernel[inputname] = suite_input = BenchmarkGroup() for (fname, f) in functions - si[fname] = @benchmarkable $f($kf, $kargs, $X, $Y) + suite_input[fname] = @benchmarkable $f($kf, $kargs, $X, $Y) end end end # Test the AD frameworks ## Zygote -SUITE["zygote"] = s_zygote = BenchmarkGroup() +SUITE["Zygote"] = suite_zygote = BenchmarkGroup() for (kname, (kargs, kf)) in kernels - s_zygote[kname] = sk = BenchmarkGroup() + suite_zygote[kname] = suite_kernel = BenchmarkGroup() for (inputname, (X, Y)) in inputtypes - sk[inputname] = si = BenchmarkGroup() + suite_kernel[inputname] = suite_input = BenchmarkGroup() for (fname, f) in functions - si[fname] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, Y - $f($kf(args...), x, y) + suite_input[fname] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, y + $f($kf, args, x, y) end end end end +## ForwardDiff +# Right now there is no canonical way to turn (kargs, X, Y) into an array. +# SUITE["ForwardDiff"] = suite_forwarddiff = BenchmarkGroup() +# for (kname, (kargs, kf)) in kernels +# suite_forwarddiff[kname] = suite_kernel = BenchmarkGroup() +# for (inputname, (X, Y)) in inputtypes +# suite_kernel[inputname] = suite_input = BenchmarkGroup() +# for (fname, f) in functions +# suite_input[fname] = @benchmarkable ForwardDiff.gradient($kargs, $X, $Y) do args, x, y +# $f($kf, args, x, y) +# end +# end +# end +# end + # Uncomment the following to run benchmark locally -# tune!(SUITE) +tune!(SUITE) -# results = run(SUITE, verbose=true) +results = run(SUITE, verbose=true) From 80b48458e607e980c8744548b7ded8a039a1f68d Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 15:27:44 +0100 Subject: [PATCH 03/10] Add reverse pass --- benchmark/benchmarks.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 92591d261..112cc3ca4 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -65,9 +65,15 @@ for (kname, (kargs, kf)) in kernels for (inputname, (X, Y)) in inputtypes suite_kernel[inputname] = suite_input = BenchmarkGroup() for (fname, f) in functions - suite_input[fname] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, y + # Forward-pass + suite_input[fname * "_forward"] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, y $f($kf, args, x, y) end + # Reverse pass + out, pb = Zygote.pullback(kargs, X, Y) do args, x, y + f(kf, args, x, y) + end + suite_input[fname * "_reverse"] = @benchmarkable $pb($out) end end end From 072f53fa6726bf18ec852679ab7c73bc69393fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= Date: Tue, 1 Feb 2022 15:30:55 +0100 Subject: [PATCH 04/10] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- benchmark/benchmarks.jl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 112cc3ca4..90674d577 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -26,14 +26,14 @@ kernels = Dict( "Constant" => ((2.0,), x->ConstantKernel(;c=x)), "White" => ((), ()->WhiteKernel()), # Cosine Kernel - "Cosine" => ((), ()->CosineKernel()), + "Cosine" => ((), () -> CosineKernel()), # Exponential Kernels - "Exponential" => ((), ()->ExponentialKernel()), - "Gibbs" => ((), ()->GibbsKernel(;lengthscale=sin)), - "SqExponential" => ((), ()->SqExponentialKernel()), - "GammaExponential" => ((1.0,), x->GammaExponentialKernel(;γ=2 * logistic(x))), + "Exponential" => ((), () -> ExponentialKernel()), + "Gibbs" => ((), () -> GibbsKernel(; lengthscale=sin)), + "SqExponential" => ((), () -> SqExponentialKernel()), + "GammaExponential" => ((1.0,), x -> GammaExponentialKernel(; γ=2 * logistic(x))), # Exponentiated Kernel - "Exponentiated" => ((), ()->ExponentiatedKernel()), + "Exponentiated" => ((), () -> ExponentiatedKernel()), ) inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv)) @@ -66,7 +66,9 @@ for (kname, (kargs, kf)) in kernels suite_kernel[inputname] = suite_input = BenchmarkGroup() for (fname, f) in functions # Forward-pass - suite_input[fname * "_forward"] = @benchmarkable Zygote.pullback($kargs, $X, $Y) do args, x, y + suite_input[fname * "_forward"] = @benchmarkable Zygote.pullback( + $kargs, $X, $Y + ) do args, x, y $f($kf, args, x, y) end # Reverse pass @@ -97,4 +99,4 @@ end tune!(SUITE) -results = run(SUITE, verbose=true) +results = run(SUITE; verbose=true) From 4b7590281ae0e0f6a2dd3bd9b1fa9eb201ba9598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= Date: Tue, 1 Feb 2022 15:31:03 +0100 Subject: [PATCH 05/10] Update benchmark/benchmarks.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- benchmark/benchmarks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 90674d577..ce9cbd45c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -23,8 +23,8 @@ SUITE = BenchmarkGroup() # Create a list of kernel and their constructors kernels = Dict( # Constant Kernels - "Constant" => ((2.0,), x->ConstantKernel(;c=x)), - "White" => ((), ()->WhiteKernel()), + "Constant" => ((2.0,), x -> ConstantKernel(; c=x)), + "White" => ((), () -> WhiteKernel()), # Cosine Kernel "Cosine" => ((), () -> CosineKernel()), # Exponential Kernels From a8c9b692d2b2de49c3fd80d3114b9861ecece9ff Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 15:32:04 +0100 Subject: [PATCH 06/10] Comment run --- benchmark/benchmarks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index ce9cbd45c..d0ab79294 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -97,6 +97,6 @@ end # Uncomment the following to run benchmark locally -tune!(SUITE) +# tune!(SUITE) -results = run(SUITE; verbose=true) +# results = run(SUITE; verbose=true) From 6f1817d86532d233807911f892577471d08ec471 Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 15:49:31 +0100 Subject: [PATCH 07/10] Remove ParameterHandling --- benchmark/benchmarks.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index d0ab79294..ba57c8d84 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,6 +1,5 @@ using BenchmarkTools using KernelFunctions -using ParameterHandling using LogExpFunctions: logistic using Zygote using ForwardDiff From 31994a32cad7a08ac3cbba92d97d1d1df2c766ea Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 16:10:32 +0100 Subject: [PATCH 08/10] Correct the lengthscale function --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index ba57c8d84..82d3b094f 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -28,7 +28,7 @@ kernels = Dict( "Cosine" => ((), () -> CosineKernel()), # Exponential Kernels "Exponential" => ((), () -> ExponentialKernel()), - "Gibbs" => ((), () -> GibbsKernel(; lengthscale=sin)), + "Gibbs" => ((), () -> GibbsKernel(; lengthscale=x->sin.(x))), "SqExponential" => ((), () -> SqExponentialKernel()), "GammaExponential" => ((1.0,), x -> GammaExponentialKernel(; γ=2 * logistic(x))), # Exponentiated Kernel From 27dd8266c6ad120c4b4f4a59dc800de21d7366fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Galy-Fajou?= Date: Tue, 1 Feb 2022 16:23:28 +0100 Subject: [PATCH 09/10] Update benchmark/benchmarks.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 82d3b094f..90c969114 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -28,7 +28,7 @@ kernels = Dict( "Cosine" => ((), () -> CosineKernel()), # Exponential Kernels "Exponential" => ((), () -> ExponentialKernel()), - "Gibbs" => ((), () -> GibbsKernel(; lengthscale=x->sin.(x))), + "Gibbs" => ((), () -> GibbsKernel(; lengthscale=x -> sin.(x))), "SqExponential" => ((), () -> SqExponentialKernel()), "GammaExponential" => ((1.0,), x -> GammaExponentialKernel(; γ=2 * logistic(x))), # Exponentiated Kernel From 066630d07e3d0c8213bb1cdae74ec12b4a375c48 Mon Sep 17 00:00:00 2001 From: Theo Galy-Fajou Date: Tue, 1 Feb 2022 16:24:00 +0100 Subject: [PATCH 10/10] Remove Gibbs --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 90c969114..cb9d59f22 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -28,7 +28,7 @@ kernels = Dict( "Cosine" => ((), () -> CosineKernel()), # Exponential Kernels "Exponential" => ((), () -> ExponentialKernel()), - "Gibbs" => ((), () -> GibbsKernel(; lengthscale=x -> sin.(x))), + # "Gibbs" => ((), () -> GibbsKernel(; lengthscale=x -> sin.(x))), "SqExponential" => ((), () -> SqExponentialKernel()), "GammaExponential" => ((1.0,), x -> GammaExponentialKernel(; γ=2 * logistic(x))), # Exponentiated Kernel