Merge branch 'main' into krc/no_empty_varargs

.github/workflows/CI.yml

@@ -2,10 +2,10 @@ name: CI
 on:
   pull_request:
     branches:
-      - master
+      - main
   push:
     branches:
-      - master
+      - main
     tags: '*'
 jobs:
   test:
@@ -52,6 +52,8 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v5
         with:
-          file: lcov.info
+          files: lcov.info
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/Documentation.yml

@@ -3,7 +3,7 @@ name: Documentation
 on:
   push:
     branches:
-      - master
+      - main
     tags: '*'
 
 jobs:

README.md

@@ -7,8 +7,8 @@
 
 [![][docs-stable-img]][docs-stable-url]
 [![][docs-dev-img]][docs-dev-url]
-[![Build Status](https://github.com/JuliaCI/BenchmarkTools.jl/workflows/CI/badge.svg)](https://github.com/JuliaCI/BenchmarkTools.jl/actions/workflows/CI.yml?query=branch%3Amaster)
-[![Code Coverage](https://codecov.io/gh/JuliaCI/BenchmarkTools.jl/branch/master/graph/badge.svg?label=codecov&token=ccN7NZpkBx)](https://codecov.io/gh/JuliaCI/BenchmarkTools.jl)
+[![Build Status](https://github.com/JuliaCI/BenchmarkTools.jl/workflows/CI/badge.svg)](https://github.com/JuliaCI/BenchmarkTools.jl/actions/workflows/CI.yml?query=branch%3Amain)
+[![Code Coverage](https://codecov.io/gh/JuliaCI/BenchmarkTools.jl/branch/main/graph/badge.svg?label=codecov&token=ccN7NZpkBx)](https://codecov.io/gh/JuliaCI/BenchmarkTools.jl)
 [![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle)
 [![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
 

docs/src/manual.md

@@ -57,12 +57,11 @@ BenchmarkTools.Trial: 10000 samples with 1000 evaluations.
  Memory estimate: 0 bytes, allocs estimate: 0.
 ```
 
-Alternatively, you can use the `@btime` or `@belapsed` macros.
-These take exactly the same arguments as `@benchmark`, but
-behave like the `@time` or `@elapsed` macros included with
-Julia: `@btime` prints the minimum time and memory allocation
-before returning the value of the expression, while `@belapsed`
-returns the minimum time in seconds.
+Alternatively, you can use the `@btime`, `@btimed`,
+`@belapsed`, `@ballocated`, or `@ballocations` macros. These
+take exactly the same arguments as `@benchmark`, but behave
+like the `@time`, `@timed`, `@elapsed`, `@allocated`, or
+`@allocations` macros included with Julia.
 
 ```julia
 julia> @btime sin(1)
@@ -71,6 +70,15 @@ julia> @btime sin(1)
 
 julia> @belapsed sin(1)
 1.3614228456913828e-8
+
+julia> @btimed sin(1)
+(value = 0.8414709848078965, time = 9.16e-10, bytes = 0, alloc = 0, gctime = 0.0)
+
+julia> @ballocated rand(4, 4)
+208
+
+julia> @ballocations rand(4, 4)
+2
 ```
 
 ### Benchmark `Parameters`

src/BenchmarkTools.jl

@@ -63,7 +63,16 @@ export BenchmarkGroup,
 
 include("execution.jl")
 
-export tune!, warmup, @ballocated, @benchmark, @benchmarkable, @belapsed, @btime, @bprofile
+export tune!,
+    warmup,
+    @ballocated,
+    @ballocations,
+    @benchmark,
+    @benchmarkable,
+    @belapsed,
+    @btime,
+    @btimed,
+    @bprofile
 
 #################
 # Serialization #

src/execution.jl

@@ -641,6 +641,30 @@ macro ballocated(args...)
     )
 end
 
+"""
+    @ballocations expression [other parameters...]
+
+Similar to the `@allocations` macro included with Julia,
+this macro evaluates an expression, discarding the resulting
+value, and returns the total number of allocations made
+during its execution.
+
+Unlike `@allocations`, it uses the `@benchmark` macro from
+the `BenchmarkTools` package, and accepts all of the same
+additional parameters as `@benchmark`. The returned number
+of allocations corresponds to the trial with the *minimum*
+elapsed time measured during the benchmark.
+"""
+macro ballocations(args...)
+    return esc(
+        quote
+            $BenchmarkTools.allocs(
+                $BenchmarkTools.minimum($BenchmarkTools.@benchmark $(args...))
+            )
+        end,
+    )
+end
+
 """
     @btime expression [other parameters...]
 
@@ -684,6 +708,46 @@ macro btime(args...)
     )
 end
 
+"""
+    @btimed expression [other parameters...]
+
+Similar to the `@timed` macro included with Julia, this
+macro executes an expression and returns a `NamedTuple`
+containing the value of the expression, the minimum elapsed
+time in seconds, the total bytes allocated, the number of
+allocations, and the garbage collection time in seconds
+during the benchmark.
+
+Unlike `@timed`, it uses the `@benchmark` macro from the
+`BenchmarkTools` package for more detailed and consistent
+performance measurements. The elapsed time reported is the
+minimum time measured during the benchmark. It accepts all
+additional parameters supported by `@benchmark`.
+"""
+macro btimed(args...)
+    _, params = prunekwargs(args...)
+    bench, trial, result = gensym(), gensym(), gensym()
+    trialmin = gensym()
+    tune_phase = hasevals(params) ? :() : :($BenchmarkTools.tune!($bench))
+    return esc(
+        quote
+            local $bench = $BenchmarkTools.@benchmarkable $(args...)
+            $tune_phase
+            local $trial, $result = $BenchmarkTools.run_result(
+                $bench; warmup=$(hasevals(params))
+            )
+            local $trialmin = $BenchmarkTools.minimum($trial)
+            (
+                value=$result,
+                time=$BenchmarkTools.time($trialmin) / 1e9,  # `@timed` macro returns elapsed time in seconds
+                bytes=$BenchmarkTools.memory($trialmin),
+                alloc=$BenchmarkTools.allocs($trialmin),
+                gctime=$BenchmarkTools.gctime($trialmin) / 1e9,
+            )
+        end,
+    )
+end
+
 """
     @bprofile expression [other parameters...]
 

src/groups.jl

@@ -121,6 +121,10 @@ allocs(group::BenchmarkGroup) = mapvals(allocs, group)
 params(group::BenchmarkGroup) = mapvals(params, group)
 
 ratio(group::BenchmarkGroup, groups::BenchmarkGroup...) = mapvals(ratio, group, groups...)
+
+"""
+    judge(target::BenchmarkGroup, baseline::BenchmarkGroup; [time_tolerance::Float64=0.05])
+"""
 function judge(group::BenchmarkGroup, groups::BenchmarkGroup...; kwargs...)
     return mapvals((x...) -> judge(x...; kwargs...), group, groups...)
 end

src/trials.jl

@@ -176,6 +176,11 @@ function ratio(a::Real, b::Real)
     return Float64(a / b)
 end
 
+"""
+    ratio(target::TrialEstimate, baseline::TrialEstimate)
+
+Returns a ratio of the target estimate to the baseline estimate, as e.g. `time(target)/time(baseline)`.
+"""
 function ratio(a::TrialEstimate, b::TrialEstimate)
     ttol = max(params(a).time_tolerance, params(b).time_tolerance)
     mtol = max(params(a).memory_tolerance, params(b).memory_tolerance)
@@ -218,8 +223,17 @@ memory(t::TrialJudgement) = t.memory
 ratio(t::TrialJudgement) = t.ratio
 params(t::TrialJudgement) = params(ratio(t))
 
-judge(a::TrialEstimate, b::TrialEstimate; kwargs...) = judge(ratio(a, b); kwargs...)
+"""
+    judge(target::TrialEstimate, baseline::TrialEstimate; [time_tolerance::Float64=0.05])
+
+Report on whether the first estimate `target` represents a regression or an improvement with respect to the second estimate `baseline`.
+"""
+judge(target::TrialEstimate, baseline::TrialEstimate; kwargs...) =
+    judge(ratio(target, baseline); kwargs...)
 
+"""
+    judge(r::TrialRatio, [time_tolerance::Float64=0.05])
+"""
 function judge(r::TrialRatio; kwargs...)
     newr = copy(r)
     newr.params = Parameters(params(r); kwargs...)
@@ -377,7 +391,7 @@ function Base.show(io::IO, ::MIME"text/plain", t::Trial)
         else
             ""
         end,
-        ".\n",
+        " per sample.\n",
     )
 
     perm = sortperm(t.times)

test/ExecutionTests.jl

@@ -308,8 +308,8 @@ b = @bprofile likegcd(x, y) setup = (x = rand(2:200); y = rand(2:200))
 io = IOBuffer()
 Profile.print(IOContext(io, :displaysize => (24, 200)))
 str = String(take!(io))
-@test occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; #?_run", str)
-@test !occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+; #?tune!", str)
+@test occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+[; ] #?_run", str)
+@test !occursin(r"BenchmarkTools(\.jl)?(/|\\)src(/|\\)execution\.jl:\d+[; ] #?tune!", str)
 b = @bprofile 1 + 1
 Profile.print(IOContext(io, :displaysize => (24, 200)))
 str = String(take!(io))
@@ -345,6 +345,22 @@ str = String(take!(io))
 @test @ballocated(sin(0)) == 0
 @test @ballocated(Ref(1)) == 2 * sizeof(Int)  # 1 for the pointer, 1 for content
 
+@test @ballocations(sin($(foo.x)), evals = 3, samples = 10, setup = (foo.x = 0)) == 0
+@test @ballocations(sin(0)) == 0
+@test @ballocations(Ref(1)) == 1
+
+@test let stats = @btimed sin($(foo.x)) evals = 3 samples = 10 setup = (foo.x = 0)
+    stats.value == sin(0) &&
+        stats.time > 0 &&
+        stats.bytes == 0 &&
+        stats.alloc == 0 &&
+        stats.gctime == 0
+end
+
+@test let stats = @btimed Ref(1)
+    stats.bytes > 0 && stats.alloc == 1 && stats.gctime == 0
+end
+
 let fname = tempname()
     try
         ret = open(fname, "w") do f

test/TrialsTests.jl

@@ -268,7 +268,7 @@ end
 
 trial = BenchmarkTools.Trial(BenchmarkTools.Parameters(), [1.0, 1.01], [0.0, 0.0], 0, 0)
 @test sprint(show, "text/plain", trial) == """
-BenchmarkTools.Trial: 2 samples with 1 evaluation.
+BenchmarkTools.Trial: 2 samples with 1 evaluation per sample.
  Range (min … max):  1.000 ns … 1.010 ns  ┊ GC (min … max): 0.00% … 0.00%
  Time  (median):     1.005 ns             ┊ GC (median):    0.00%
  Time  (mean ± σ):   1.005 ns ± 0.007 ns  ┊ GC (mean ± σ):  0.00% ± 0.00%