@@ -1,0 +1,468 @@
defmodule Mix.Tasks.Exgo.Bench do
@shortdoc "Storage/read-path benchmark across backends + native git (perf source-of-truth)"
@moduledoc """
Times common git read operations against a backend (or native `git`) and emits
one table column. The intent is a stable source-of-truth for the read-path perf
effort: same operations, same fixture, comparable numbers across
* Filesystem (local disk)
* Filesystem on a CephFS mount
* S3 (MinIO)
* S3 (Ceph RGW)
* native `git` (the reference — what's achievable)
Each operation is timed `--runs` times (median reported). For the
ExGitObjectstore backends the per-process pack cache is cleared before every
run, so each measurement is cold. Native `git` runs a fresh process per run
(warm OS page cache — noted; the gaps we care about dwarf cache effects).
## Produce a column
# Filesystem / CephFS (root contains repos/<id>):
mix exgo.bench --target filesystem --root /data --repo <uuid> \\
--label "FS (local)" --out fs.json
# S3 (MinIO or Ceph RGW):
mix exgo.bench --target s3 --bucket anvil-git-packed --repo <uuid> \\
--s3-host localhost --s3-port 9000 --s3-key minioadmin --s3-secret minioadmin \\
--label "S3 / MinIO" --out minio.json
# Native git (reference column); --git-dir is a real git repo:
mix exgo.bench --target git --git-dir /data/repos/<uuid> --label "native git" --out git.json
Pass an explicit fixture so every column measures identical inputs:
--head <sha> --old <sha> --file <path> --blob <sha> --subdir <path>
(omit to auto-derive from the repo; the orchestrator script derives once and
passes them to all columns).
## Render the table
mix exgo.bench --render fs.json cephfs.json minio.json ceph.json git.json
"""
use Mix.Task
alias ExGitObjectstore.{Diff, ObjectResolver, Repo}
@switches [
target: :string,
root: :string,
repo: :string,
git_dir: :string,
bucket: :string,
s3_host: :string,
s3_port: :integer,
s3_key: :string,
s3_secret: :string,
s3_scheme: :string,
label: :string,
out: :string,
runs: :integer,
op_timeout: :integer,
depth: :integer,
head: :string,
old: :string,
file: :string,
blob: :string,
subdir: :string,
render: :boolean
]
# Canonical operation order (rows). Each: id, label, :elixir runner, :git args.
# Runners get the fixture map; they should return a term or raise. Defined as a
# function (not a module attribute) so the closures can call module privates.
defp ops do
[
%{
id: :resolve_head,
label: "resolve HEAD",
elixir: fn r, _f -> ok!(ExGitObjectstore.resolve(r, "HEAD")) end,
git: fn _f -> ["rev-parse", "HEAD"] end
},
%{
id: :read_commit,
label: "read 1 commit",
elixir: fn r, f -> ok!(ObjectResolver.read(r, f.head)) end,
git: fn f -> ["cat-file", "-p", f.head] end
},
%{
id: :read_blob,
label: "read 1 blob",
elixir: fn r, f -> ok!(ExGitObjectstore.blob(r, f.blob)) end,
git: fn f -> ["cat-file", "-p", f.blob] end
},
%{
id: :open_file,
label: "open file by path",
elixir: fn r, f -> open_file(r, f) end,
git: fn f -> ["show", "#{f.head}:#{f.file}"] end
},
%{
id: :list_root,
label: "list root tree",
elixir: fn r, f -> ok!(ExGitObjectstore.tree(r, f.head, "/")) end,
git: fn f -> ["ls-tree", f.head] end
},
%{
id: :tree_sizes,
label: "tree + blob sizes",
elixir: fn r, f -> tree_with_sizes(r, f) end,
git: fn f -> ["ls-tree", "-l", f.head] end
},
%{
id: :log_50,
label: "commit log page (50)",
elixir: fn r, f -> ok!(ExGitObjectstore.log_page(r, f.head, limit: 50)) end,
git: fn f -> ["log", "-n", "50", "--format=%H%n%an%n%s", f.head] end
},
%{
id: :log_full,
label: "full history walk",
elixir: fn r, f -> ok!(ExGitObjectstore.log(r, f.head, [])) end,
git: fn f -> ["rev-list", "--count", f.head] end
},
%{
id: :diff,
label: "diff two commits",
elixir: fn r, f -> ok!(Diff.diff_commits(r, f.old, f.head)) end,
git: fn f -> ["diff", f.old, f.head] end
},
%{
id: :ahead_behind,
label: "ahead/behind",
elixir: fn r, f -> ok!(ExGitObjectstore.ahead_behind(r, f.old, f.head)) end,
git: fn f -> ["rev-list", "--left-right", "--count", "#{f.old}...#{f.head}"] end
},
%{
id: :merge_base,
label: "merge-base",
elixir: fn r, f -> ok!(ExGitObjectstore.merge_base(r, f.old, f.head)) end,
git: fn f -> ["merge-base", f.old, f.head] end
},
%{
id: :clone,
label: "clone (read all reachable)",
elixir: fn r, f -> clone_walk(r, f.head) end,
git: fn f -> [:pipe_pack, f.head] end
}
]
end
@impl Mix.Task
def run(argv) do
{opts, files, _} = OptionParser.parse(argv, switches: @switches)
if opts[:render] do
render(files)
else
bench_column(opts)
end
end
# ── Column run ──────────────────────────────────────────────────────────────
defp bench_column(opts) do
target = opts[:target] || Mix.raise("--target filesystem|s3|git required")
runs = opts[:runs] || 3
op_timeout = opts[:op_timeout] || 60_000
ctx = build_ctx(target, opts)
Mix.shell().info("== #{opts[:label] || target} (#{target}) ==")
results = Enum.map(ops(), &run_op(&1, target, ctx, runs, op_timeout))
write_json(opts, target, runs, results)
end
defp run_op(op, target, ctx, runs, op_timeout) do
# clone reads the whole repo and is slow on our walk — one run is enough.
op_runs = if op.id == :clone, do: 1, else: runs
{ms, timed_out} =
case measure(target, op, ctx, ctx.fixture, op_runs, op_timeout) do
{:timeout, t} -> {Float.round(t / 1000, 1), true}
ms -> {ms, false}
end
prefix = if timed_out, do: ">", else: ""
Mix.shell().info(" #{String.pad_trailing(op.label, 28)} #{prefix}#{fmt(ms)}")
%{"id" => Atom.to_string(op.id), "label" => op.label, "ms" => ms, "timeout" => timed_out}
end
defp write_json(opts, target, runs, results) do
payload = %{
"label" => opts[:label] || target,
"target" => target,
"repo" => opts[:repo] || opts[:git_dir],
"runs" => runs,
"results" => results
}
case opts[:out] do
nil ->
:ok
path ->
File.write!(path, Jason.encode!(payload, pretty: true))
Mix.shell().info("wrote #{path}")
end
end
defp measure(target, op, ctx, fixture, runs, timeout) do
attempts = for _ <- 1..runs, do: one_attempt(target, op, ctx, fixture, timeout)
if Enum.any?(attempts, &(&1 == :timeout)) do
{:timeout, timeout * 1000}
else
median_ms(attempts)
end
end
defp one_attempt("git", op, ctx, fixture, timeout) do
with_timeout(fn -> run_git(ctx.git_dir, op.git.(fixture)) end, timeout)
end
# Each attempt runs in a fresh Task (a fresh process → cold per-process pack
# cache, naturally), bounded by `timeout` ms so a pathologically slow op (e.g.
# clone on a huge repo via our walk) records a sentinel instead of hanging.
defp one_attempt(_backend, op, ctx, fixture, timeout) do
with_timeout(
fn ->
try do
op.elixir.(ctx.repo, fixture)
rescue
e -> {:bench_error, Exception.message(e)}
end
end,
timeout
)
end
defp with_timeout(fun, timeout) do
task = Task.async(fn -> time_us(fun) end)
case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do
{:ok, us} -> us
_ -> :timeout
end
end
# ── Operation helpers ────────────────────────────────────────────────────────
defp ok!({:ok, v}), do: v
# log_page/3 returns {:ok, commits, cursor}.
defp ok!({:ok, v, _cursor}), do: v
defp ok!(other), do: raise("unexpected: #{inspect(other)}")
defp open_file(repo, f) do
{:ok, %{tree: tree_sha}} = ObjectResolver.read(repo, f.head)
{:ok, %{entries: entries}} = ObjectResolver.read(repo, tree_sha)
entry = Enum.find(entries, &(&1.name == f.file)) || hd(entries)
ok!(ExGitObjectstore.blob(repo, entry.sha))
end
defp tree_with_sizes(repo, f) do
{:ok, tree} = ExGitObjectstore.tree(repo, f.head, "/")
blob_shas = for e <- tree.entries, String.starts_with?(e.mode, "100"), do: e.sha
ok!(ExGitObjectstore.blob_sizes(repo, blob_shas))
end
defp clone_walk(repo, head) do
walk(repo, head, MapSet.new()) |> MapSet.size()
end
defp walk(repo, sha, seen) do
if MapSet.member?(seen, sha), do: seen, else: visit(repo, sha, MapSet.put(seen, sha))
end
defp visit(repo, sha, seen) do
case ObjectResolver.read(repo, sha) do
{:ok, %ExGitObjectstore.Object.Commit{tree: t, parents: ps}} ->
Enum.reduce(ps, walk(repo, t, seen), &walk(repo, &1, &2))
{:ok, %ExGitObjectstore.Object.Tree{entries: es}} ->
Enum.reduce(es, seen, fn %{sha: s}, acc -> walk(repo, s, acc) end)
{:ok, %ExGitObjectstore.Object.Tag{object: o}} ->
walk(repo, o, seen)
_ ->
seen
end
end
# native git: clone serving = read+pack all reachable from HEAD.
defp run_git(dir, [:pipe_pack, head]) do
{_, 0} =
System.shell(
"git -C #{dir} rev-list #{head} | git -C #{dir} pack-objects --stdout --revs >/dev/null",
stderr_to_stdout: true
)
end
defp run_git(dir, args) do
{_, 0} = System.cmd("git", ["-C", dir | args], stderr_to_stdout: true)
end
# ── Context + fixture ────────────────────────────────────────────────────────
defp build_ctx("git", opts) do
dir = opts[:git_dir] || Mix.raise("--git-dir required for --target git")
%{git_dir: dir, repo: nil, fixture: fixture(opts, {:git, dir})}
end
defp build_ctx(target, opts) when target in ["filesystem", "s3"] do
if target == "s3", do: {:ok, _} = Application.ensure_all_started(:ex_aws_s3)
repo_id = opts[:repo] || Mix.raise("--repo required")
repo = Repo.new(repo_id, storage: storage(target, opts))
%{git_dir: nil, repo: repo, fixture: fixture(opts, {:repo, repo})}
end
defp storage("filesystem", opts) do
{ExGitObjectstore.Storage.Filesystem,
%{root: Path.expand(opts[:root] || Mix.raise("--root required"))}}
end
defp storage("s3", opts) do
{ExGitObjectstore.Storage.S3,
%{
bucket: opts[:bucket] || Mix.raise("--bucket required"),
ex_aws_config: [
access_key_id: opts[:s3_key] || "minioadmin",
secret_access_key: opts[:s3_secret] || "minioadmin",
region: "us-east-1",
host: opts[:s3_host] || "localhost",
port: opts[:s3_port] || 9000,
scheme: opts[:s3_scheme] || "http://"
]
}}
end
# Use explicit fixture flags when present, else derive from the source. Only
# derives when something is missing (derivation reads the repo; passing all
# five flags — what the orchestrator does — skips it entirely).
defp fixture(opts, source) do
provided = %{
head: opts[:head],
old: opts[:old],
file: opts[:file],
blob: opts[:blob],
subdir: opts[:subdir]
}
if Enum.all?(Map.values(provided), &(&1 != nil)) do
provided
else
base = derive_fixture(source, opts[:depth] || 50)
Map.merge(base, for({k, v} <- provided, v != nil, into: %{}, do: {k, v}))
end
end
defp derive_fixture({:git, dir}, depth) do
head = git_out(dir, ["rev-parse", "HEAD"])
old = git_out(dir, ["rev-parse", "HEAD~#{depth}"])
ls = git_out(dir, ["ls-tree", head]) |> String.split("\n", trim: true)
{blob, file} = first_blob_from_ls(ls)
%{head: head, old: old, file: file, blob: blob, subdir: first_dir_from_ls(ls)}
end
defp derive_fixture({:repo, repo}, depth) do
{:ok, head} = ExGitObjectstore.resolve(repo, "HEAD")
entries = log_entries(ExGitObjectstore.log_page(repo, head, limit: depth + 1))
old = entries |> List.last() |> commit_sha() || head
{:ok, tree} = ExGitObjectstore.tree(repo, head, "/")
blob_entry = Enum.find(tree.entries, &String.starts_with?(&1.mode, "100"))
dir_entry = Enum.find(tree.entries, &String.starts_with?(&1.mode, "40"))
%{
head: head,
old: old || head,
file: (blob_entry && blob_entry.name) || "README.md",
blob: (blob_entry && blob_entry.sha) || head,
subdir: (dir_entry && dir_entry.name) || ""
}
end
defp log_entries({:ok, entries, _cursor}) when is_list(entries), do: entries
defp commit_sha({sha, _commit}) when is_binary(sha), do: sha
defp commit_sha(%{sha: sha}), do: sha
defp commit_sha(%{"sha" => sha}), do: sha
defp commit_sha(_), do: nil
defp first_blob_from_ls(lines) do
line = Enum.find(lines, &String.contains?(&1, " blob ")) || hd(lines)
[meta, name] = String.split(line, "\t", parts: 2)
sha = meta |> String.split() |> Enum.at(2)
{sha, name}
end
defp first_dir_from_ls(lines) do
case Enum.find(lines, &String.contains?(&1, " tree ")) do
nil -> ""
line -> line |> String.split("\t", parts: 2) |> List.last()
end
end
defp git_out(dir, args) do
{out, 0} = System.cmd("git", ["-C", dir | args], stderr_to_stdout: true)
String.trim(out)
end
# ── Timing ───────────────────────────────────────────────────────────────────
defp time_us(fun) do
{us, _} = :timer.tc(fun)
us
end
defp median_ms(times_us) do
sorted = Enum.sort(times_us)
n = length(sorted)
mid = div(n, 2)
us =
if rem(n, 2) == 1,
do: Enum.at(sorted, mid),
else: (Enum.at(sorted, mid - 1) + Enum.at(sorted, mid)) / 2
Float.round(us / 1000, 1)
end
defp fmt(ms), do: "#{ms} ms"
# ── Render ───────────────────────────────────────────────────────────────────
defp render(files) do
cols = Enum.map(files, fn f -> Jason.decode!(File.read!(f)) end)
rows =
Enum.map(ops(), fn op ->
id = Atom.to_string(op.id)
cells = Enum.map(cols, fn c -> cell(c, id) end)
{op.label, cells}
end)
headers = ["operation" | Enum.map(cols, & &1["label"])]
Mix.shell().info(table_line(headers))
Mix.shell().info(table_sep(length(headers)))
Enum.each(rows, fn {label, cells} ->
Mix.shell().info(table_line([label | Enum.map(cells, &cell_str/1)]))
end)
end
defp cell(col, id) do
case Enum.find(col["results"], &(&1["id"] == id)) do
%{"ms" => ms, "timeout" => true} -> {:timeout, ms}
%{"ms" => ms} -> ms
_ -> nil
end
end
defp cell_str({:timeout, ms}), do: ">#{ms}"
defp cell_str(ms) when is_number(ms), do: "#{ms}"
defp cell_str(_), do: "—"
defp table_line(cells), do: "| " <> Enum.join(cells, " | ") <> " |"
defp table_sep(n), do: "|" <> String.duplicate(" --- |", n)
end