Basic reference_job feature

Basis is from #281 which we let lie for too long, sorry.

Also too much had changed in the mean time (`RelativeStatistics`)
so reopening it didn't make a ton of sesnse.

Fixes #179

Author: PragTob

lib/benchee/configuration.ex

@@ -39,7 +39,8 @@ defmodule Benchee.Configuration do
             after_scenario: nil,
             measure_function_call_overhead: false,
             title: nil,
-            profile_after: false
+            profile_after: false,
+            reference_job: nil
 
   @typedoc """
   The configuration supplied by the user as either a map or a keyword list
@@ -133,6 +134,7 @@ defmodule Benchee.Configuration do
       [`:cprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Cprof.html),
       [`:eprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Eprof.html) and
       [`:fprof`](https://hexdocs.pm/mix/Mix.Tasks.Profile.Fprof.html).
+    * `:reference_job` - Set which function in the benchmark all others will be compared against.
   """
   @type user_configuration :: map | keyword
 
@@ -163,7 +165,8 @@ defmodule Benchee.Configuration do
           after_scenario: Hooks.hook_function() | nil,
           measure_function_call_overhead: boolean,
           title: String.t() | nil,
-          profile_after: boolean | atom | {atom, keyword}
+          profile_after: boolean | atom | {atom, keyword},
+          reference_job: String.t() | nil
         }
 
   @time_keys [:time, :warmup, :memory_time, :reduction_time]

lib/benchee/relative_statistics.ex

@@ -9,7 +9,7 @@ defmodule Benchee.RelativeStatistics do
   has to happen before they are loaded to avoid recalculating their statistics.
   """
 
-  alias Benchee.{Scenario, Statistics, Suite}
+  alias Benchee.{Configuration, Scenario, Statistics, Suite}
 
   @doc """
   Calculate the statistics of scenarios relative to each other and sorts scenarios.
@@ -22,17 +22,17 @@ defmodule Benchee.RelativeStatistics do
   """
   @spec relative_statistics(Suite.t()) :: Suite.t()
   def relative_statistics(suite) do
-    %Suite{suite | scenarios: calculate_relative_statistics(suite.scenarios)}
+    %Suite{suite | scenarios: calculate_relative_statistics(suite.scenarios, suite.configuration)}
   end
 
-  defp calculate_relative_statistics([]), do: []
+  defp calculate_relative_statistics([], _config), do: []
 
-  defp calculate_relative_statistics(scenarios) do
+  defp calculate_relative_statistics(scenarios, config) do
     scenarios
     |> scenarios_by_input()
     |> Enum.flat_map(fn scenarios_with_same_input ->
       sorted_scenarios = sort(scenarios_with_same_input)
-      {reference, others} = split_reference_scenario(sorted_scenarios)
+      {reference, others} = split_reference_scenario(sorted_scenarios, config)
       others_with_relative = statistics_relative_to(others, reference)
       [reference | others_with_relative]
     end)
@@ -59,9 +59,25 @@ defmodule Benchee.RelativeStatistics do
     end)
   end
 
-  # right now we take the first scenario as we sorted them and it is the fastest,
-  # whenever we implement #179 though this becomes more involved
-  defp split_reference_scenario(scenarios) do
+  defp split_reference_scenario(scenarios, config)
+
+  defp split_reference_scenario(scenarios, %Configuration{reference_job: reference_job})
+       when is_binary(reference_job) do
+    split_scenarios =
+      Enum.split_with(scenarios, fn scenario -> scenario.name == reference_job end)
+
+    case split_scenarios do
+      {[reference_scenario], others} -> {reference_scenario, others}
+      _reference_not_found -> fastest_as_reference(scenarios)
+    end
+  end
+
+  # no reference_job configured? Just take the first one, as this is post sort it'll be the fastet
+  defp split_reference_scenario(scenarios, _config) do
+    fastest_as_reference(scenarios)
+  end
+
+  defp fastest_as_reference(scenarios) do
     [reference | others] = scenarios
     {reference, others}
   end

test/benchee/relative_statistics_test.exs

@@ -1,7 +1,7 @@
 defmodule Benchee.RelativeStatistcsTest do
   use ExUnit.Case, async: true
 
-  alias Benchee.{CollectionData, Scenario, Statistics, Suite}
+  alias Benchee.{CollectionData, Configuration, Scenario, Statistics, Suite}
   import Benchee.RelativeStatistics
 
   describe "computing relative statistics" do
@@ -43,7 +43,7 @@ defmodule Benchee.RelativeStatistcsTest do
       assert stats.relative_less == nil
     end
 
-    test "handles the fastest value being zerio alright" do
+    test "handles the fastest value being zero alright" do
       suite = %Suite{
         scenarios: [
           scenario_with_average(0.0),
@@ -98,6 +98,55 @@ defmodule Benchee.RelativeStatistcsTest do
       assert stats2.relative_less == 1.0
       assert stats2.absolute_difference == 0.0
     end
+
+    test "setting the reference job to be the slowest" do
+      suite = %Suite{
+        scenarios: [
+          named_scenario_with_average("A", 394.0, nil),
+          named_scenario_with_average("B", 14.0, nil)
+        ],
+        configuration: %Configuration{reference_job: "A"}
+      }
+
+      suite = relative_statistics(suite)
+      # reference scenario always comes out on top even if slowest
+      assert [a_stats, b_stats] = stats_from(suite)
+
+      assert a_stats.relative_more == nil
+      assert a_stats.relative_less == nil
+      assert a_stats.absolute_difference == nil
+
+      assert b_stats.absolute_difference == -380.0
+      assert_in_delta b_stats.relative_more, 0.0355, 0.001
+      assert_in_delta b_stats.relative_less, 28.14, 0.1
+    end
+
+    test "sandwich reference job" do
+      suite = %Suite{
+        scenarios: [
+          named_scenario_with_average("A", 1.0, nil),
+          named_scenario_with_average("B", 2.0, nil),
+          named_scenario_with_average("C", 3.0, nil)
+        ],
+        configuration: %Configuration{reference_job: "B"}
+      }
+
+      suite = relative_statistics(suite)
+      # reference scenario always comes out on top even if slowest
+      assert [b_stats, a_stats, c_stats] = stats_from(suite)
+
+      assert b_stats.relative_more == nil
+      assert b_stats.relative_less == nil
+      assert b_stats.absolute_difference == nil
+
+      assert a_stats.absolute_difference == -1.0
+      assert_in_delta a_stats.relative_more, 0.5, 0.001
+      assert_in_delta a_stats.relative_less, 2.0, 0.01
+
+      assert c_stats.absolute_difference == 1.0
+      assert_in_delta c_stats.relative_more, 1.5, 0.001
+      assert_in_delta c_stats.relative_less, 0.66, 0.01
+    end
   end
 
   describe "sorting behaviour" do
@@ -138,6 +187,24 @@ defmodule Benchee.RelativeStatistcsTest do
 
       assert Enum.map(sorted, fn scenario -> scenario.name end) == ["1", "2", "3"]
     end
+
+    test "given a configured reference_job returns that as the first regardless of performance" do
+      fourth = named_scenario_with_average("4", 400.1, nil)
+      second = named_scenario_with_average("2", 200.0, nil)
+      third = named_scenario_with_average("3", 400.0, nil)
+      first = named_scenario_with_average("1", 100.0, nil)
+      reference = named_scenario_with_average("Ref", 500.0, nil)
+
+      scenarios = [fourth, second, third, first, reference]
+
+      sorted =
+        relative_statistics(%Suite{
+          scenarios: scenarios,
+          configuration: %Configuration{reference_job: "Ref"}
+        }).scenarios
+
+      assert Enum.map(sorted, fn scenario -> scenario.name end) == ["Ref", "1", "2", "3", "4"]
+    end
   end
 
   describe "dealing correctly with different inputs" do
@@ -187,6 +254,35 @@ defmodule Benchee.RelativeStatistcsTest do
              ]
     end
 
+    test "calculate the correct relatives based on input names even given a reference_job" do
+      suite = %Suite{
+        scenarios: [
+          named_input_scenario_with_average("A", "1", 100.0),
+          named_input_scenario_with_average("B", "1", 200.0),
+          named_input_scenario_with_average("A", "10", 1_000.0),
+          named_input_scenario_with_average("B", "10", 10_000.0)
+        ],
+        configuration: %Configuration{reference_job: "B"}
+      }
+
+      suite = relative_statistics(suite)
+
+      stats =
+        suite.scenarios
+        |> Enum.map(
+          &{&1.name, &1.input_name, &1.run_time_data.statistics.absolute_difference,
+           &1.run_time_data.statistics.relative_more}
+        )
+        |> Enum.sort()
+
+      assert stats == [
+               {"A", "1", -100.0, 0.5},
+               {"A", "10", -9_000, 0.1},
+               {"B", "1", nil, nil},
+               {"B", "10", nil, nil}
+             ]
+    end
+
     test "calculate the correct relatives based on different ordering" do
       suite = %Suite{
         scenarios: [