Just have Benchee in all environments.

mix.exs

@@ -112,7 +112,7 @@ defmodule Pleroma.Mixfile do
       {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
-      {:benchee, "~> 1.0", only: [:dev, :test]}
+      {:benchee, "~> 1.0"}
     ] ++ oauth_deps
   end