warm up at the beginning of every measurement (#330)

* warm up at the beginning of every measurement.

* use JuliaFormatter(Benchmarktools; overwrite=true, style=BlueStyle())

* deprecate warmup instead of remove

* JuliaFormatter

* ensure that only one eval is used for the warmup, test deprecations

* format

* export warmup

* with formatting

* hmm

* add a boolean to run kwargs

* cool

* warmup in the case evals are unspecified.

* clean up regex to optionally match gensymed defs from Julia 1.0

* oops, rusty regex

src/execution.jl

@@ -106,12 +106,15 @@ end
 # Note that trials executed via `run` and `lineartrial` are always executed at top-level
 # scope, in order to allow transfer of locally-scoped variables into benchmark scope.
 
-function _run(b::Benchmark, p::Parameters; verbose=false, pad="", kwargs...)
+function _run(b::Benchmark, p::Parameters; verbose=false, pad="", warmup=true, kwargs...)
     params = Parameters(p; kwargs...)
     @assert params.seconds > 0.0 "time limit must be greater than 0.0"
     params.gctrial && gcscrub()
     start_time = Base.time()
     trial = Trial(params)
+    if warmup
+        b.samplefunc(b.quote_vals, Parameters(params; evals=1)) #warmup sample
+    end
     params.gcsample && gcscrub()
     s = b.samplefunc(b.quote_vals, params)
     push!(trial, s[1:(end - 1)]...)
@@ -180,6 +183,8 @@ function _lineartrial(b::Benchmark, p::Parameters=b.params; maxevals=RESOLUTION,
     params = Parameters(p; kwargs...)
     estimates = zeros(maxevals)
     completed = 0
+    params.evals = 1
+    b.samplefunc(b.quote_vals, params) #warmup sample
     params.gctrial && gcscrub()
     start_time = time()
     for evals in eachindex(estimates)
@@ -193,6 +198,10 @@ function _lineartrial(b::Benchmark, p::Parameters=b.params; maxevals=RESOLUTION,
 end
 
 function warmup(item; verbose::Bool=true)
+    Base.depwarn(
+        "`warmup` is deprecated because `run` now warms up every time automatically",
+        :warmup,
+    )
     return run(item; verbose=verbose, samples=1, evals=1, gctrial=false, gcsample=false)
 end
 
@@ -288,7 +297,6 @@ function tune!(
     kwargs...,
 )
     if !p.evals_set
-        warmup(b; verbose=false)
         estimate = ceil(Int, minimum(lineartrial(b, p; kwargs...)))
         b.params.evals = guessevals(estimate)
     end
@@ -436,9 +444,8 @@ macro benchmark(args...)
     return esc(
         quote
             local $tmp = $BenchmarkTools.@benchmarkable $(args...)
-            $BenchmarkTools.warmup($tmp)
             $(hasevals(params) ? :() : :($BenchmarkTools.tune!($tmp)))
-            $BenchmarkTools.run($tmp)
+            $BenchmarkTools.run($tmp; warmup=$(hasevals(params)))
         end,
     )
 end
@@ -656,9 +663,10 @@ macro btime(args...)
     return esc(
         quote
             local $bench = $BenchmarkTools.@benchmarkable $(args...)
-            $BenchmarkTools.warmup($bench)
             $tune_phase
-            local $trial, $result = $BenchmarkTools.run_result($bench)
+            local $trial, $result = $BenchmarkTools.run_result(
+                $bench; warmup=$(hasevals(params))
+            )
             local $trialmin = $BenchmarkTools.minimum($trial)
             local $trialallocs = $BenchmarkTools.allocs($trialmin)
             println(
@@ -704,10 +712,18 @@ macro bprofile(args...)
     return esc(
         quote
             local $tmp = $BenchmarkTools.@benchmarkable $(args...)
-            $BenchmarkTools.warmup($tmp)
-            $(hasevals(params) ? :() : :($BenchmarkTools.tune!($tmp)))
+            $(
+                if hasevals(params)
+                    :(run(
+                        $tmp, $BenchmarkTools.Parameters($tmp.params; evals=1); warmup=false
+                    ))
+                else
+                    :($BenchmarkTools.tune!($tmp))
+                end
+            )
             $BenchmarkTools.Profile.clear()
-            $BenchmarkTools.@profile $BenchmarkTools.run($tmp)
+            #TODO: improve @bprofile to only measure the running code and none of the setup
+            $BenchmarkTools.@profile $BenchmarkTools.run($tmp, $tmp.params; warmup=false)
         end,
     )
 end

test/ExecutionTests.jl

@@ -148,12 +148,38 @@ tune!(b_pass)
 # warmup #
 ###########
 
-p = params(warmup(@benchmarkable sin(1)))
+@test_deprecated warmup(@benchmarkable sin(1))
+
+is_warm = false
+function needs_warm()
+    global is_warm
+    if is_warm
+        sleep(0.1)
+    else
+        sleep(2)
+        is_warm = true
+    end
+end
+
+w = @benchmarkable needs_warm()
+w.params.seconds = 1
+
+#test that all measurements from lineartrial used in tune! are warm
+is_warm = false
+@test maximum(BenchmarkTools.lineartrial(w, w.params)) < 1e9
+
+#test that run warms up the benchmark
+tune!(w)
+is_warm = false
+@test minimum(run(w).times) < 1e9
+
+#test that belapsed warms up the benchmark
+is_warm = false
+@test (@belapsed needs_warm() seconds = 1) < 1
 
-@test p.samples == 1
-@test p.evals == 1
-@test p.gctrial == false
-@test p.gcsample == false
+#test that belapsed warms up the benchmark even when evals are set
+is_warm = false
+@test (@belapsed needs_warm() seconds = 1 evals = 1) < 1
 
 ##############
 # @benchmark #
@@ -282,9 +308,8 @@ b = @bprofile likegcd(x, y) setup = (x = rand(2:200); y = rand(2:200))
 io = IOBuffer()
 Profile.print(IOContext(io, :displaysize => (24, 200)))
 str = String(take!(io))
-@test occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; _run", str)
-@test !occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; warmup", str)
-@test !occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; tune!", str)
+@test occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; #?_run", str)
+@test !occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; #?tune!", str)
 b = @bprofile 1 + 1
 Profile.print(IOContext(io, :displaysize => (24, 200)))
 str = String(take!(io))