▸
bench/graph_build.exs
+148
−0
@@ -1,0 +1,148 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Baseline benchmark for PR 1 of the commit-graph work (ex_git_objectstore#26).
#
# Measures the cost of the current per-commit-object walker on a 5000-commit
# linear chain, and times Graph.Builder.build/1 on the same repo. The walker
# being measured is the one replicated from Anvil's
# lib/anvil/git/objectstore.ex `collect_ancestors` (the hot path that has been
# saturating prod CPU — see fangorn/anvil#55).
#
# Run with: mix run bench/graph_build.exs
alias ExGitObjectstore.{Graph, Object}
alias ExGitObjectstore.Object.{Commit, Tree}
alias ExGitObjectstore.{ObjectResolver, Repo, Storage.Memory}
defmodule Bench.Walker do
@moduledoc false
# Replicates Anvil.Git.Objectstore.collect_ancestors/3 — reads each commit
# via ObjectResolver.read/2 and enqueues its parents. Purely a baseline to
# compare against; not intended for production use.
def walk_all_ancestors(repo, tip) do
do_walk(repo, [tip], MapSet.new())
end
defp do_walk(_repo, [], visited), do: visited
defp do_walk(repo, [sha | rest], visited) do
if MapSet.member?(visited, sha) do
do_walk(repo, rest, visited)
else
visited = MapSet.put(visited, sha)
parents =
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{parents: ps}} -> ps
_ -> []
end
do_walk(repo, parents ++ rest, visited)
end
end
end
defmodule Bench.Fixture do
@moduledoc false
def build_linear_chain(repo, n) do
{:ok, tree_sha} = Object.write(repo, Tree.new([]))
Enum.reduce(1..n, nil, fn i, prev ->
parents = if prev, do: [prev], else: []
ident = "A <a@a.com> #{1_700_000_000 + i} +0000"
commit = %Commit{
tree: tree_sha,
parents: parents,
author: ident,
committer: ident,
message: "c\n"
}
{:ok, sha} = Object.write(repo, commit)
sha
end)
end
end
defmodule Bench.Run do
@moduledoc false
def now_us, do: :erlang.monotonic_time(:microsecond)
def time(label, fun) do
t0 = now_us()
result = fun.()
t1 = now_us()
elapsed_ms = (t1 - t0) / 1000
padded = String.pad_trailing(label, 45)
IO.puts(" #{padded} #{Float.round(elapsed_ms, 2) |> :erlang.float_to_binary(decimals: 2)} ms")
{result, elapsed_ms}
end
def run(n) do
IO.puts("\n=== Commit-graph baseline — #{n} commits, linear chain ===\n")
{:ok, mem_pid} = Memory.start_link()
repo = Repo.new("bench", storage: {Memory, Memory.config(mem_pid)})
:ok = ExGitObjectstore.init(repo)
tip =
time("Fixture: write #{n} commits", fn ->
Bench.Fixture.build_linear_chain(repo, n)
end)
|> elem(0)
:ok = ExGitObjectstore.create_branch(repo, "main", tip)
# 1) Today's walker (pure Elixir cat_object-per-commit BFS).
{visited, walker_ms} =
time("Walker: collect_ancestors (tip→roots)", fn ->
Bench.Walker.walk_all_ancestors(repo, tip)
end)
IO.puts(" (walker visited #{MapSet.size(visited)} commits)")
# 2) PR 1's builder.
{{:ok, graph}, build_ms} =
time("Builder.build (from all refs)", fn -> Graph.build(repo) end)
IO.puts(" (graph contains #{Graph.size(graph)} commits)")
# 3) Persist + reload (future fast path — no cat_object calls).
{:ok, load_ms} =
time("Graph.save + Graph.load (persist + rehydrate)", fn ->
:ok = Graph.save(repo, graph)
{:ok, _} = Graph.load(repo)
:ok
end)
|> then(fn {_, ms} -> {:ok, ms} end)
IO.puts("\n walker_ms / build_ms = #{Float.round(walker_ms / max(build_ms, 0.1), 2)}x")
IO.puts(" walker_ms / load_ms = #{Float.round(walker_ms / max(load_ms, 0.1), 2)}x")
IO.puts("")
end
end
# Default: 5000 commits, override via CLI arg.
n =
case System.argv() do
[arg | _] -> String.to_integer(arg)
_ -> 5_000
end
Bench.Run.run(n)
▸
lib/ex_git_objectstore.ex
+9
−0
@@ -43,6 +43,15 @@
* `merge_base/3` — lowest common ancestor of two commits
* `ancestor?/3` — true if A is an ancestor of B
## Commit-graph index
`ExGitObjectstore.Graph` provides an optional persisted commit-graph
index with topological generation numbers and corrected commit dates.
Once built and saved (`Graph.build/1`, `Graph.save/2`), it is loaded
wholesale into memory for fast ancestry / ahead-behind queries without
per-commit object reads. See that module and the `Graph.BinaryFormat`
moduledoc for details.
"""
alias ExGitObjectstore.{Merge, Object, ObjectResolver, Ref, Repo, Walk}
▸
lib/ex_git_objectstore/graph.ex
+137
−0
@@ -1,0 +1,137 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph do
@moduledoc """
In-memory commit-graph index: one `Entry` per commit reachable from refs,
enriched with a topological generation number and a corrected committer
date (git's "topologically consistent committer date"). Walks read the
graph once from storage, then traverse in-memory — no per-parent object
lookups.
Binary format is defined in `ExGitObjectstore.Graph.BinaryFormat`. The
graph is persisted as a single blob per repo at `graph/commit-graph.v1`.
This module owns the struct, persistence, and a small lookup API. Building
from refs lives in `ExGitObjectstore.Graph.Builder`.
"""
alias ExGitObjectstore.Graph.{BinaryFormat, Builder, Entry}
alias ExGitObjectstore.Repo
@blob_key "graph/commit-graph.v1"
@type sha :: String.t()
@type t :: %__MODULE__{
version: non_neg_integer(),
shas: [sha()],
by_sha: %{sha() => Entry.t()}
}
defstruct version: 1, shas: [], by_sha: %{}
@doc """
Storage blob key under which the graph is persisted.
"""
@spec blob_key() :: String.t()
def blob_key, do: @blob_key
@doc """
Build a fresh graph for `repo` by walking every ref. Delegates to
`ExGitObjectstore.Graph.Builder`. Does not persist — call `save/2`
separately.
"""
@spec build(Repo.t()) :: {:ok, t()} | {:error, term()}
def build(%Repo{} = repo), do: Builder.build(repo)
@doc """
Load and deserialize the commit-graph for `repo`. Returns
`{:error, :missing}` if the graph has not been built yet, or
`{:error, reason}` if the blob is corrupt.
"""
@spec load(Repo.t()) :: {:ok, t()} | {:error, :missing | term()}
def load(%Repo{} = repo) do
case Repo.storage_call(repo, :get_blob, [@blob_key]) do
{:ok, bin} -> BinaryFormat.deserialize(bin)
{:error, :not_found} -> {:error, :missing}
{:error, _} = err -> err
end
end
@doc """
Serialize `graph` and write it to `repo`'s storage under the standard key.
"""
@spec save(Repo.t(), t()) :: :ok | {:error, term()}
def save(%Repo{} = repo, %__MODULE__{} = graph) do
Repo.storage_call(repo, :put_blob, [@blob_key, BinaryFormat.serialize(graph)])
end
@doc """
Remove the persisted graph. Does not touch in-memory state. Succeeds
whether or not the blob existed.
"""
@spec delete(Repo.t()) :: :ok | {:error, term()}
def delete(%Repo{} = repo) do
Repo.storage_call(repo, :delete_blob, [@blob_key])
end
@doc """
Return the topological generation number of `sha` in `graph`, or `:error`
if `sha` is not in the graph.
"""
@spec generation(t(), sha()) :: {:ok, non_neg_integer()} | :error
def generation(%__MODULE__{by_sha: by_sha}, sha) do
case Map.fetch(by_sha, sha) do
{:ok, %Entry{generation: g}} -> {:ok, g}
:error -> :error
end
end
@doc """
Return the corrected committer date of `sha`, or `:error` if `sha` is not
in the graph.
"""
@spec corrected_commit_date(t(), sha()) :: {:ok, non_neg_integer()} | :error
def corrected_commit_date(%__MODULE__{by_sha: by_sha}, sha) do
case Map.fetch(by_sha, sha) do
{:ok, %Entry{corrected_commit_date: ccd}} -> {:ok, ccd}
:error -> :error
end
end
@doc """
Return parent SHAs of `sha` in the order recorded at commit time
(first-parent is `hd/1`), or `:error` if `sha` is not in the graph.
"""
@spec parents(t(), sha()) :: {:ok, [sha()]} | :error
def parents(%__MODULE__{by_sha: by_sha}, sha) do
case Map.fetch(by_sha, sha) do
{:ok, %Entry{parents: ps}} -> {:ok, ps}
:error -> :error
end
end
@doc """
Whether `sha` has an entry in `graph`.
"""
@spec member?(t(), sha()) :: boolean()
def member?(%__MODULE__{by_sha: by_sha}, sha), do: Map.has_key?(by_sha, sha)
@doc """
Number of commits in the graph.
"""
@spec size(t()) :: non_neg_integer()
def size(%__MODULE__{by_sha: by_sha}), do: map_size(by_sha)
end
▸
lib/ex_git_objectstore/graph/binary_format.ex
+293
−0
@@ -1,0 +1,293 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph.BinaryFormat do
@moduledoc """
Binary serialization for `ExGitObjectstore.Graph`.
All integers big-endian. SHAs are stored as their 20-byte raw
representation (not the 40-character hex string).
## Layout
Header (12 bytes):
magic "ECG1" 4 bytes
version u32 (=1) 4 bytes
commit_count u32 (=N) 4 bytes
Fan-out (1024 bytes):
256 × u32 — entry i = count of OIDs with first byte ≤ i
OID table (20·N bytes):
N × 20 bytes — raw SHAs in ascending order
Offset table (4·N bytes):
N × u32 — byte offset of each entry from the start of the
entries block (same order as the OID table)
Entries block (variable):
For each commit (in OID-table order):
tree_oid 20 bytes
generation u32
corrected_commit_date u64
commit_time u64
parent_count u8
parent_indices u32 × parent_count
(each = index into OID table)
Typical size: ~45 B/commit for single-parent history (tree 20 + gen 4 +
ccd 8 + ctime 8 + count 1 + parent 4 = 45, plus 20 B oid + 4 B offset in
the tables = ~69 B total per commit).
"""
alias ExGitObjectstore.Graph
alias ExGitObjectstore.Graph.Entry
@magic "ECG1"
@version 1
@header_size 12
@fanout_size 1024
@doc """
Serialize a graph to its on-disk binary form.
"""
@spec serialize(Graph.t()) :: binary()
def serialize(%Graph{shas: shas, by_sha: by_sha}) do
# The struct carries an ordered shas list, but normalize defensively so
# serialize is robust to callers who hand us an unsorted list.
ordered = Enum.sort(shas)
n = length(ordered)
raw_shas = Enum.map(ordered, &hex_to_raw/1)
index_of = ordered |> Enum.with_index() |> Map.new()
{entries_iodata, offsets} = encode_entries(ordered, by_sha, index_of)
fanout = build_fanout(raw_shas)
header = <<@magic, @version::big-32, n::big-32>>
oid_table = IO.iodata_to_binary(raw_shas)
offset_table = IO.iodata_to_binary(for off <- offsets, do: <<off::big-32>>)
IO.iodata_to_binary([header, fanout, oid_table, offset_table, entries_iodata])
end
@doc """
Deserialize a binary into a graph. Returns `{:ok, graph}` or
`{:error, reason}`.
"""
@spec deserialize(binary()) ::
{:ok, Graph.t()} | {:error, :bad_magic | :truncated | {:unsupported_version, integer()}}
def deserialize(bin) when is_binary(bin) do
with {:ok, n, rest} <- parse_header(bin),
{:ok, _fanout, rest} <- parse_fanout(rest),
{:ok, raw_shas, rest} <- parse_oid_table(rest, n),
{:ok, offsets, rest} <- parse_offset_table(rest, n),
{:ok, entries} <- parse_entries(rest, offsets, raw_shas) do
shas = Enum.map(raw_shas, &raw_to_hex/1)
by_sha = Map.new(Enum.zip(shas, entries))
{:ok, %Graph{version: @version, shas: shas, by_sha: by_sha}}
end
end
# -- Header --
defp parse_header(bin) when byte_size(bin) < @header_size, do: {:error, :truncated}
defp parse_header(<<@magic, version::big-32, _n::big-32, _rest::binary>>)
when version != @version,
do: {:error, {:unsupported_version, version}}
defp parse_header(<<@magic, @version::big-32, n::big-32, rest::binary>>),
do: {:ok, n, rest}
defp parse_header(_), do: {:error, :bad_magic}
# -- Fan-out --
defp parse_fanout(bin) when byte_size(bin) < @fanout_size, do: {:error, :truncated}
defp parse_fanout(<<fanout::binary-size(@fanout_size), rest::binary>>),
do: {:ok, fanout, rest}
# -- OID table --
defp parse_oid_table(bin, n) do
need = 20 * n
case bin do
<<oids::binary-size(^need), rest::binary>> ->
{:ok, split_fixed(oids, 20), rest}
_ ->
{:error, :truncated}
end
end
# -- Offset table --
defp parse_offset_table(bin, n) do
need = 4 * n
case bin do
<<offsets_bin::binary-size(^need), rest::binary>> ->
offsets = for <<off::big-32 <- offsets_bin>>, do: off
{:ok, offsets, rest}
_ ->
{:error, :truncated}
end
end
# -- Entries --
#
# We walk the offset list in order; each offset is relative to the start
# of the entries block. The trailing slice is bounded by the next offset
# (or the end of the block for the last entry).
# Empty offsets → empty entries via do_decode_pairs base case.
defp parse_entries(bin, offsets, raw_shas) do
hex_by_index =
raw_shas
|> Enum.map(&raw_to_hex/1)
|> :array.from_list()
pairs = offsets_with_ends(offsets)
do_decode_pairs(bin, pairs, hex_by_index, [])
end
defp parse_entry_slice(bin, off, len, idx) when is_integer(len) do
case bin do
<<_::binary-size(^off), slice::binary-size(^len), _::binary>> ->
decode_entry(slice, idx)
_ ->
{:error, :truncated}
end
end
# Last entry: no length — slice is bin from off to end.
defp parse_entry_slice(bin, off, :rest, idx) do
case bin do
<<_::binary-size(^off), slice::binary>> -> decode_entry(slice, idx)
_ -> {:error, :truncated}
end
end
defp decode_entry(
<<tree::binary-size(20), generation::big-32, ccd::big-64, ctime::big-64, pc::big-8,
parents_bin::binary>>,
idx
) do
expected = 4 * pc
case parents_bin do
<<parent_indices::binary-size(^expected), _::binary>> ->
parent_shas = decode_parents(parent_indices, idx, [])
{:ok,
%Entry{
tree: raw_to_hex(tree),
generation: generation,
corrected_commit_date: ccd,
commit_time: ctime,
parents: parent_shas
}}
_ ->
{:error, :truncated}
end
end
defp decode_entry(_, _), do: {:error, :truncated}
defp decode_parents(<<>>, _idx, acc), do: Enum.reverse(acc)
defp decode_parents(<<i::big-32, rest::binary>>, idx, acc) do
sha = :array.get(i, idx)
decode_parents(rest, idx, [sha | acc])
end
defp offsets_with_ends([]), do: []
defp offsets_with_ends([only]), do: [{only, :rest}]
defp offsets_with_ends([a, b | rest]) do
[{a, b - a} | offsets_with_ends([b | rest])]
end
defp do_decode_pairs(_bin, [], _idx, acc), do: {:ok, Enum.reverse(acc)}
defp do_decode_pairs(bin, [{off, len} | rest], idx, acc) do
with {:ok, entry} <- parse_entry_slice(bin, off, len, idx) do
do_decode_pairs(bin, rest, idx, [entry | acc])
end
end
# -- Encoding helpers --
defp encode_entries(ordered, by_sha, index_of) do
{rev_entries, _final_offset, rev_offsets} =
Enum.reduce(ordered, {[], 0, []}, fn sha, {entries_acc, offset, offsets_acc} ->
entry = Map.fetch!(by_sha, sha)
bin = encode_entry(entry, index_of)
{[bin | entries_acc], offset + byte_size(bin), [offset | offsets_acc]}
end)
{Enum.reverse(rev_entries), Enum.reverse(rev_offsets)}
end
defp encode_entry(%Entry{} = e, index_of) do
parent_indices =
for p <- e.parents, into: <<>> do
i = Map.fetch!(index_of, p)
<<i::big-32>>
end
tree_raw = hex_to_raw(e.tree)
<<tree_raw::binary, e.generation::big-32, e.corrected_commit_date::big-64,
e.commit_time::big-64, length(e.parents)::big-8, parent_indices::binary>>
end
defp build_fanout(raw_shas) do
counts = :array.new(size: 256, default: 0, fixed: true)
counts =
Enum.reduce(raw_shas, counts, fn <<b, _::binary>>, acc ->
:array.set(b, :array.get(b, acc) + 1, acc)
end)
{fanout_iodata, _} =
Enum.reduce(0..255, {[], 0}, fn i, {acc, running} ->
running = running + :array.get(i, counts)
{[<<running::big-32>> | acc], running}
end)
fanout_iodata |> Enum.reverse() |> IO.iodata_to_binary()
end
defp split_fixed(<<>>, _size), do: []
defp split_fixed(bin, size) do
<<chunk::binary-size(^size), rest::binary>> = bin
[chunk | split_fixed(rest, size)]
end
defp hex_to_raw(hex) when is_binary(hex) and byte_size(hex) == 40 do
Base.decode16!(hex, case: :lower)
end
defp raw_to_hex(raw) when is_binary(raw) and byte_size(raw) == 20 do
Base.encode16(raw, case: :lower)
end
end
▸
lib/ex_git_objectstore/graph/builder.ex
+191
−0
@@ -1,0 +1,191 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph.Builder do
@moduledoc """
Full commit-graph builder — scans every ref in a repo, reads every
reachable commit once, and computes topological generation numbers plus
corrected committer dates.
Used both for one-time bootstrap of existing repos and for fsck-style
rebuilds. Incremental updates (for the push path) are not implemented
here; see `ExGitObjectstore.Graph` issue #26 PR plan.
"""
alias ExGitObjectstore.Graph
alias ExGitObjectstore.Graph.Entry
alias ExGitObjectstore.Object.Commit
alias ExGitObjectstore.{ObjectResolver, Repo}
@doc """
Build a fresh commit-graph for `repo` by walking all refs.
Returns `{:ok, graph}` on success. Returns `{:error, reason}` if a
reachable commit is missing or cannot be read.
Cost is O(N) `cat_object` calls, where N is the number of reachable
commits. Tips that resolve to non-commit objects (e.g. annotated tags)
are peeled before traversal.
"""
@spec build(Repo.t()) :: {:ok, Graph.t()} | {:error, term()}
def build(%Repo{} = repo) do
with {:ok, tips} <- collect_tips(repo),
{:ok, raw} <- walk_reachable(repo, tips) do
graph = compute_generations_and_ccd(raw)
{:ok, graph}
end
end
# -- Step 1: collect all tip commit SHAs from branches + tags. --
defp collect_tips(repo) do
with {:ok, branches} <- ExGitObjectstore.branches(repo),
{:ok, tags} <- ExGitObjectstore.tags(repo) do
tips =
(branches ++ tags)
|> Enum.map(fn {_name, sha} -> sha end)
|> Enum.uniq()
|> Enum.flat_map(&peel_or_drop(repo, &1))
|> Enum.uniq()
{:ok, tips}
end
end
defp peel_or_drop(repo, sha) do
case peel_to_commit(repo, sha) do
{:ok, commit_sha} -> [commit_sha]
_ -> []
end
end
defp peel_to_commit(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{}} -> {:ok, sha}
{:ok, %ExGitObjectstore.Object.Tag{object: target}} -> peel_to_commit(repo, target)
{:ok, _other} -> {:error, :not_a_commit}
{:error, _} = err -> err
end
end
# -- Step 2: BFS the DAG from tips, reading each commit once. --
defp walk_reachable(repo, tips) do
do_walk(repo, tips, %{})
end
defp do_walk(_repo, [], raw), do: {:ok, raw}
defp do_walk(repo, [sha | rest], raw) do
if Map.has_key?(raw, sha) do
do_walk(repo, rest, raw)
else
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{} = commit} ->
entry_raw = %{
tree: commit.tree,
parents: commit.parents,
commit_time: parse_timestamp(commit.committer)
}
do_walk(repo, commit.parents ++ rest, Map.put(raw, sha, entry_raw))
{:error, _} = err ->
err
end
end
end
# -- Step 3: topological generation + CCD in a single Kahn's-algorithm
# pass over the reverse adjacency (parents → children). --
defp compute_generations_and_ccd(raw) do
shas = Map.keys(raw)
children_of = build_children_index(raw)
in_deg = Map.new(shas, fn sha -> {sha, length(raw[sha].parents)} end)
roots = for {sha, 0} <- in_deg, do: sha
{gen, ccd} = topo_process(raw, children_of, in_deg, roots, %{}, %{})
entries =
Map.new(raw, fn {sha, %{tree: tree, parents: parents, commit_time: ct}} ->
{sha,
%Entry{
tree: tree,
parents: parents,
generation: Map.fetch!(gen, sha),
corrected_commit_date: Map.fetch!(ccd, sha),
commit_time: ct
}}
end)
%Graph{version: 1, shas: Enum.sort(shas), by_sha: entries}
end
defp build_children_index(raw) do
Enum.reduce(raw, %{}, fn {child_sha, %{parents: parents}}, acc ->
Enum.reduce(parents, acc, fn parent_sha, acc2 ->
Map.update(acc2, parent_sha, [child_sha], &[child_sha | &1])
end)
end)
end
defp topo_process(_raw, _children_of, _in_deg, [], gen, ccd), do: {gen, ccd}
defp topo_process(raw, children_of, in_deg, [sha | rest], gen, ccd) do
%{parents: parents, commit_time: ct} = Map.fetch!(raw, sha)
my_gen =
case parents do
[] -> 1
_ -> 1 + Enum.max(Enum.map(parents, &Map.fetch!(gen, &1)))
end
my_ccd =
case parents do
[] -> ct
_ -> Enum.max([ct | Enum.map(parents, &Map.fetch!(ccd, &1))])
end
gen = Map.put(gen, sha, my_gen)
ccd = Map.put(ccd, sha, my_ccd)
children = Map.get(children_of, sha, [])
{in_deg, newly_ready} =
Enum.reduce(children, {in_deg, []}, fn child, {deg_acc, ready_acc} ->
new_deg = Map.fetch!(deg_acc, child) - 1
deg_acc = Map.put(deg_acc, child, new_deg)
if new_deg == 0 do
{deg_acc, [child | ready_acc]}
else
{deg_acc, ready_acc}
end
end)
topo_process(raw, children_of, in_deg, rest ++ newly_ready, gen, ccd)
end
# "Name <email> <timestamp> <tz>" — match the trailing unix timestamp.
defp parse_timestamp(str) when is_binary(str) do
case Regex.run(~r/(\d+)\s+[+-]\d{4}$/, str) do
[_, ts] -> String.to_integer(ts)
_ -> 0
end
end
defp parse_timestamp(_), do: 0
end
▸
lib/ex_git_objectstore/graph/entry.ex
+32
−0
@@ -1,0 +1,32 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph.Entry do
@moduledoc """
One commit's entry in the commit-graph index.
"""
@type sha :: String.t()
@type t :: %__MODULE__{
tree: sha(),
parents: [sha()],
generation: non_neg_integer(),
corrected_commit_date: non_neg_integer(),
commit_time: non_neg_integer()
}
@enforce_keys [:tree, :parents, :generation, :corrected_commit_date, :commit_time]
defstruct [:tree, :parents, :generation, :corrected_commit_date, :commit_time]
end
▸
lib/ex_git_objectstore/storage.ex
+13
−0
@@ -62,4 +62,17 @@
@callback get_head(config, prefix) :: {:ok, String.t()} | {:error, term()}
@callback put_head(config, prefix, target :: String.t()) :: :ok | {:error, term()}
# -- Side-index blobs (not git objects / refs / packs) --
#
# Generic key/value slot for ancillary data such as commit-graph indexes.
# Keys are repo-relative, slash-separated paths (e.g. "graph/commit-graph.v1")
# and must not contain ".." segments. Implementations should reject traversal.
@type blob_key :: String.t()
@callback get_blob(config, prefix, blob_key) :: {:ok, binary()} | {:error, term()}
@callback put_blob(config, prefix, blob_key, binary()) :: :ok | {:error, term()}
@callback delete_blob(config, prefix, blob_key) :: :ok | {:error, term()}
@callback blob_exists?(config, prefix, blob_key) :: boolean()
end
▸
lib/ex_git_objectstore/storage/filesystem.ex
+46
−0
@@ -361,11 +361,57 @@
atomic_write(path, target <> "\n")
end
# -- Side-index blobs --
@impl true
def get_blob(config, prefix, blob_key) do
path = blob_path(config, prefix, blob_key)
case File.read(path) do
{:ok, data} -> {:ok, data}
{:error, :enoent} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
@impl true
def put_blob(config, prefix, blob_key, data) do
atomic_write(blob_path(config, prefix, blob_key), data)
end
@impl true
def delete_blob(config, prefix, blob_key) do
case File.rm(blob_path(config, prefix, blob_key)) do
:ok -> :ok
{:error, :enoent} -> :ok
{:error, reason} -> {:error, reason}
end
end
@impl true
def blob_exists?(config, prefix, blob_key) do
File.exists?(blob_path(config, prefix, blob_key))
end
# -- Private --
defp object_path(config, prefix, sha) do
<<dir::binary-size(2), rest::binary>> = sha
safe_path(config.root, Path.join([prefix, "objects", dir, rest]))
end
defp blob_path(config, prefix, blob_key) do
validate_blob_key!(blob_key)
safe_path(config.root, Path.join([prefix, "blobs", blob_key]))
end
defp validate_blob_key!(blob_key) do
if not is_binary(blob_key) or blob_key == "" or String.contains?(blob_key, "..") or
String.starts_with?(blob_key, "/") do
raise ArgumentError, "invalid blob key: #{inspect(blob_key)}"
end
:ok
end
defp pack_path(config, prefix, pack_sha, ext) do
▸
lib/ex_git_objectstore/storage/memory.ex
+46
−1
@@ -28,6 +28,6 @@
@spec start_link() :: {:ok, pid()}
def start_link do
Agent.start_link(fn ->
%{objects: %{}, refs: %{}, packs: %{}, head: %{}}
%{objects: %{}, refs: %{}, packs: %{}, head: %{}, blobs: %{}}
end)
end
@@ -210,8 +210,53 @@
:ok
end
# -- Side-index blobs --
@impl true
def get_blob(%{pid: pid}, prefix, blob_key) do
validate_blob_key!(blob_key)
case Agent.get(pid, &get_in(&1, [:blobs, key(prefix, blob_key)])) do
nil -> {:error, :not_found}
data -> {:ok, data}
end
end
@impl true
def put_blob(%{pid: pid}, prefix, blob_key, data) do
validate_blob_key!(blob_key)
Agent.update(pid, &put_in(&1, [:blobs, key(prefix, blob_key)], data))
:ok
end
@impl true
def delete_blob(%{pid: pid}, prefix, blob_key) do
validate_blob_key!(blob_key)
Agent.update(pid, fn state ->
update_in(state, [:blobs], &Map.delete(&1, key(prefix, blob_key)))
end)
:ok
end
@impl true
def blob_exists?(%{pid: pid}, prefix, blob_key) do
validate_blob_key!(blob_key)
Agent.get(pid, fn state -> Map.has_key?(state.blobs, key(prefix, blob_key)) end)
end
# -- Private --
defp key(prefix, path), do: "#{prefix}/#{path}"
defp pack_key(prefix, pack_sha, ext), do: "#{prefix}/pack/#{pack_sha}.#{ext}"
defp validate_blob_key!(blob_key) do
if not is_binary(blob_key) or blob_key == "" or String.contains?(blob_key, "..") or
String.starts_with?(blob_key, "/") do
raise ArgumentError, "invalid blob key: #{inspect(blob_key)}"
end
:ok
end
end
▸
lib/ex_git_objectstore/storage/s3.ex
+39
−0
@@ -318,6 +318,45 @@
safe_key("#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}")
end
# -- Side-index blobs --
@impl true
def get_blob(config, prefix, blob_key) do
s3_get(config, blob_object_key(prefix, blob_key))
end
@impl true
def put_blob(config, prefix, blob_key, data) do
s3_put(config, blob_object_key(prefix, blob_key), data)
end
@impl true
def delete_blob(config, prefix, blob_key) do
s3_delete(config, blob_object_key(prefix, blob_key))
end
@impl true
def blob_exists?(config, prefix, blob_key) do
case s3_head(config, blob_object_key(prefix, blob_key)) do
:ok -> true
{:error, _} -> false
end
end
defp blob_object_key(prefix, blob_key) do
validate_blob_key!(blob_key)
safe_key("#{prefix}/blobs/#{blob_key}")
end
defp validate_blob_key!(blob_key) do
if not is_binary(blob_key) or blob_key == "" or String.contains?(blob_key, "..") or
String.starts_with?(blob_key, "/") do
raise ArgumentError, "invalid blob key: #{inspect(blob_key)}"
end
:ok
end
# Prevent path traversal in S3 keys
defp safe_key(key) do
if String.contains?(key, "..") do
▸
test/ex_git_objectstore/graph/binary_format_test.exs
+209
−0
@@ -1,0 +1,209 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph.BinaryFormatTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.Graph
alias ExGitObjectstore.Graph.{BinaryFormat, Entry}
@tree String.duplicate("1", 40)
defp sha(char), do: String.duplicate(char, 40)
defp entry(tree \\ @tree, parents, gen, ccd, ctime) do
%Entry{
tree: tree,
parents: parents,
generation: gen,
corrected_commit_date: ccd,
commit_time: ctime
}
end
defp make_graph(by_sha) do
%Graph{
version: 1,
shas: Enum.sort(Map.keys(by_sha)),
by_sha: by_sha
}
end
describe "round-trip" do
test "empty graph" do
g = make_graph(%{})
bin = BinaryFormat.serialize(g)
assert {:ok, g2} = BinaryFormat.deserialize(bin)
assert g2.shas == []
assert g2.by_sha == %{}
assert g2.version == 1
end
test "single root commit" do
by_sha = %{sha("a") => entry([], 1, 1000, 1000)}
g = make_graph(by_sha)
bin = BinaryFormat.serialize(g)
assert {:ok, g2} = BinaryFormat.deserialize(bin)
assert g2.shas == [sha("a")]
e = g2.by_sha[sha("a")]
assert e.parents == []
assert e.generation == 1
assert e.corrected_commit_date == 1000
assert e.commit_time == 1000
assert e.tree == @tree
end
test "linear chain of 3 commits" do
a = sha("a")
b = sha("b")
c = sha("c")
by_sha = %{
a => entry([], 1, 1000, 1000),
b => entry([a], 2, 2000, 2000),
c => entry([b], 3, 3000, 3000)
}
g = make_graph(by_sha)
bin = BinaryFormat.serialize(g)
{:ok, g2} = BinaryFormat.deserialize(bin)
assert g2.by_sha[a].parents == []
assert g2.by_sha[b].parents == [a]
assert g2.by_sha[c].parents == [b]
assert g2.by_sha[c].generation == 3
end
test "merge commit (two parents)" do
a = sha("a")
b = sha("b")
m = sha("c")
by_sha = %{
a => entry([], 1, 1000, 1000),
b => entry([], 1, 1500, 1500),
m => entry([a, b], 2, 2000, 2000)
}
g = make_graph(by_sha)
{:ok, g2} = BinaryFormat.deserialize(BinaryFormat.serialize(g))
assert Enum.sort(g2.by_sha[m].parents) == Enum.sort([a, b])
assert g2.by_sha[m].generation == 2
end
test "octopus merge (many parents) preserves order" do
p1 = sha("1")
p2 = sha("2")
p3 = sha("3")
p4 = sha("4")
p5 = sha("5")
m = sha("a")
by_sha = %{
p1 => entry([], 1, 100, 100),
p2 => entry([], 1, 100, 100),
p3 => entry([], 1, 100, 100),
p4 => entry([], 1, 100, 100),
p5 => entry([], 1, 100, 100),
m => entry([p1, p2, p3, p4, p5], 2, 200, 200)
}
g = make_graph(by_sha)
{:ok, g2} = BinaryFormat.deserialize(BinaryFormat.serialize(g))
# Parent order is semantically significant in git (first-parent walks)
assert g2.by_sha[m].parents == [p1, p2, p3, p4, p5]
end
test "large u64 corrected_commit_date preserved" do
a = sha("a")
# Far-future timestamp
ccd = 9_999_999_999
by_sha = %{a => entry([], 1, ccd, ccd)}
g = make_graph(by_sha)
{:ok, g2} = BinaryFormat.deserialize(BinaryFormat.serialize(g))
assert g2.by_sha[a].corrected_commit_date == ccd
assert g2.by_sha[a].commit_time == ccd
end
test "binary is deterministic — same graph produces same bytes" do
a = sha("a")
b = sha("b")
by_sha = %{
a => entry([], 1, 1000, 1000),
b => entry([a], 2, 2000, 2000)
}
g = make_graph(by_sha)
assert BinaryFormat.serialize(g) == BinaryFormat.serialize(g)
end
end
describe "header" do
test "starts with magic ECG1 and version 1" do
g = make_graph(%{sha("a") => entry([], 1, 100, 100)})
bin = BinaryFormat.serialize(g)
assert <<"ECG1", 0, 0, 0, 1, _count::big-32, _rest::binary>> = bin
end
test "deserialize rejects wrong magic" do
bogus = <<"XXXX", 0, 0, 0, 1, 0, 0, 0, 0>> <> :binary.copy(<<0>>, 1024)
assert {:error, :bad_magic} = BinaryFormat.deserialize(bogus)
end
test "deserialize rejects unsupported version" do
bogus = <<"ECG1", 0, 0, 0, 99, 0, 0, 0, 0>> <> :binary.copy(<<0>>, 1024)
assert {:error, {:unsupported_version, 99}} = BinaryFormat.deserialize(bogus)
end
test "deserialize rejects truncated input" do
assert {:error, :truncated} = BinaryFormat.deserialize(<<"ECG", 0>>)
end
end
describe "fan-out table" do
test "fanout[i] = count of SHAs with first byte ≤ i" do
# Three SHAs starting with 0x10, 0x20, 0xF0
s10 = "10" <> String.duplicate("0", 38)
s20 = "20" <> String.duplicate("0", 38)
sf0 = "f0" <> String.duplicate("0", 38)
by_sha = %{
s10 => entry([], 1, 1, 1),
s20 => entry([], 1, 1, 1),
sf0 => entry([], 1, 1, 1)
}
g = make_graph(by_sha)
bin = BinaryFormat.serialize(g)
<<_header::binary-size(12), fanout::binary-size(1024), _::binary>> = bin
# Extract fanout entries
fan = for <<x::big-32 <- fanout>>, do: x
assert Enum.at(fan, 0x0F) == 0
assert Enum.at(fan, 0x10) == 1
assert Enum.at(fan, 0x1F) == 1
assert Enum.at(fan, 0x20) == 2
assert Enum.at(fan, 0xEF) == 2
assert Enum.at(fan, 0xF0) == 3
assert Enum.at(fan, 0xFF) == 3
end
end
end
▸
test/ex_git_objectstore/graph/graph_test.exs
+235
−0
@@ -1,0 +1,235 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.GraphTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.{Graph, Object}
alias ExGitObjectstore.Object.{Commit, Tree}
alias ExGitObjectstore.Test.RepoHelper
# --- fixture helpers ---
defp init_repo do
repo = RepoHelper.memory_repo("graph-test-#{:erlang.unique_integer([:positive])}")
ExGitObjectstore.init(repo)
repo
end
defp empty_tree_sha(repo) do
tree = Tree.new([])
{:ok, sha} = Object.write(repo, tree)
sha
end
defp make_commit(repo, tree_sha, parents, ts, message \\ "c") do
ident = "A <a@a.com> #{ts} +0000"
commit = %Commit{
tree: tree_sha,
parents: parents,
author: ident,
committer: ident,
message: message <> "\n"
}
{:ok, sha} = Object.write(repo, commit)
sha
end
# --- tests ---
describe "build/1 — empty repo" do
test "produces an empty graph when no refs exist" do
repo = init_repo()
{:ok, graph} = Graph.build(repo)
assert Graph.size(graph) == 0
assert graph.shas == []
end
end
describe "build/1 — single branch, linear history" do
test "generation numbers are 1, 2, 3 along the chain" do
repo = init_repo()
tree = empty_tree_sha(repo)
c1 = make_commit(repo, tree, [], 1000)
c2 = make_commit(repo, tree, [c1], 2000)
c3 = make_commit(repo, tree, [c2], 3000)
:ok = ExGitObjectstore.create_branch(repo, "main", c3)
{:ok, g} = Graph.build(repo)
assert Graph.size(g) == 3
assert {:ok, 1} = Graph.generation(g, c1)
assert {:ok, 2} = Graph.generation(g, c2)
assert {:ok, 3} = Graph.generation(g, c3)
end
test "parents are recorded as hex shas, ordered" do
repo = init_repo()
tree = empty_tree_sha(repo)
c1 = make_commit(repo, tree, [], 1000)
c2 = make_commit(repo, tree, [c1], 2000)
:ok = ExGitObjectstore.create_branch(repo, "main", c2)
{:ok, g} = Graph.build(repo)
assert {:ok, []} = Graph.parents(g, c1)
assert {:ok, [^c1]} = Graph.parents(g, c2)
end
test "corrected_commit_date equals commit_time for non-skewed chain" do
repo = init_repo()
tree = empty_tree_sha(repo)
c1 = make_commit(repo, tree, [], 1000)
c2 = make_commit(repo, tree, [c1], 2000)
:ok = ExGitObjectstore.create_branch(repo, "main", c2)
{:ok, g} = Graph.build(repo)
assert {:ok, 1000} = Graph.corrected_commit_date(g, c1)
assert {:ok, 2000} = Graph.corrected_commit_date(g, c2)
end
end
describe "build/1 — CCD handles clock skew" do
test "child with earlier commit_time still gets CCD ≥ parent CCD" do
repo = init_repo()
tree = empty_tree_sha(repo)
# Parent's committer time is LATER than child's — clock skew scenario
parent = make_commit(repo, tree, [], 5000)
child = make_commit(repo, tree, [parent], 1000)
:ok = ExGitObjectstore.create_branch(repo, "main", child)
{:ok, g} = Graph.build(repo)
assert {:ok, 5000} = Graph.corrected_commit_date(g, parent)
# CCD(child) = max(commit_time(child), CCD(parent)) = max(1000, 5000) = 5000
assert {:ok, 5000} = Graph.corrected_commit_date(g, child)
# But commit_time stays the original 1000 (stored separately)
assert g.by_sha[child].commit_time == 1000
end
end
describe "build/1 — merge commit" do
test "generation is max(parent.gen)+1, CCD handles both parents" do
repo = init_repo()
tree = empty_tree_sha(repo)
root = make_commit(repo, tree, [], 100)
left = make_commit(repo, tree, [root], 200)
right1 = make_commit(repo, tree, [root], 300)
right2 = make_commit(repo, tree, [right1], 400)
merge = make_commit(repo, tree, [left, right2], 500)
:ok = ExGitObjectstore.create_branch(repo, "main", merge)
{:ok, g} = Graph.build(repo)
assert {:ok, 1} = Graph.generation(g, root)
assert {:ok, 2} = Graph.generation(g, left)
assert {:ok, 2} = Graph.generation(g, right1)
assert {:ok, 3} = Graph.generation(g, right2)
# merge.gen = max(left.gen=2, right2.gen=3) + 1 = 4
assert {:ok, 4} = Graph.generation(g, merge)
assert {:ok, parents} = Graph.parents(g, merge)
assert parents == [left, right2]
end
end
describe "build/1 — multi-ref reachability" do
test "all reachable commits across branches and tags are indexed" do
repo = init_repo()
tree = empty_tree_sha(repo)
root = make_commit(repo, tree, [], 100)
a1 = make_commit(repo, tree, [root], 200)
b1 = make_commit(repo, tree, [root], 300)
tagged = make_commit(repo, tree, [root], 400)
:ok = ExGitObjectstore.create_branch(repo, "main", a1)
:ok = ExGitObjectstore.create_branch(repo, "feature", b1)
:ok = ExGitObjectstore.create_tag(repo, "v1", tagged)
{:ok, g} = Graph.build(repo)
expected = Enum.sort([root, a1, b1, tagged])
assert Enum.sort(g.shas) == expected
end
test "commits reachable from multiple refs appear only once" do
repo = init_repo()
tree = empty_tree_sha(repo)
root = make_commit(repo, tree, [], 100)
tip = make_commit(repo, tree, [root], 200)
:ok = ExGitObjectstore.create_branch(repo, "main", tip)
:ok = ExGitObjectstore.create_branch(repo, "other", tip)
{:ok, g} = Graph.build(repo)
assert Graph.size(g) == 2
end
end
describe "save + load round-trip" do
test "load after save returns an equivalent graph" do
repo = init_repo()
tree = empty_tree_sha(repo)
c1 = make_commit(repo, tree, [], 100)
c2 = make_commit(repo, tree, [c1], 200)
c3 = make_commit(repo, tree, [c1, c2], 300)
:ok = ExGitObjectstore.create_branch(repo, "main", c3)
{:ok, original} = Graph.build(repo)
:ok = Graph.save(repo, original)
{:ok, loaded} = Graph.load(repo)
assert Enum.sort(loaded.shas) == Enum.sort(original.shas)
assert loaded.by_sha == original.by_sha
end
test "load before save returns :missing" do
repo = init_repo()
assert {:error, :missing} = Graph.load(repo)
end
test "delete removes persisted graph" do
repo = init_repo()
tree = empty_tree_sha(repo)
c = make_commit(repo, tree, [], 1)
:ok = ExGitObjectstore.create_branch(repo, "main", c)
{:ok, g} = Graph.build(repo)
:ok = Graph.save(repo, g)
assert {:ok, _} = Graph.load(repo)
:ok = Graph.delete(repo)
assert {:error, :missing} = Graph.load(repo)
end
end
describe "lookup API edge cases" do
test "generation/parents/CCD return :error for unknown sha" do
repo = init_repo()
{:ok, g} = Graph.build(repo)
missing = String.duplicate("0", 40)
assert :error = Graph.generation(g, missing)
assert :error = Graph.parents(g, missing)
assert :error = Graph.corrected_commit_date(g, missing)
refute Graph.member?(g, missing)
end
end
end
▸
test/ex_git_objectstore/graph/random_dag_test.exs
+184
−0
@@ -1,0 +1,184 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Graph.RandomDAGTest do
@moduledoc """
Pseudo-property tests: build random commit DAGs of varied shape and size,
then assert every invariant end-to-end — build, serialize, deserialize,
lookup — without external property libs.
Determinism note: each iteration seeds `:rand` from the ExUnit seed plus
the iteration index so failures are reproducible via `mix test --seed N`.
"""
use ExUnit.Case, async: true
alias ExGitObjectstore.{Graph, Object}
alias ExGitObjectstore.Graph.BinaryFormat
alias ExGitObjectstore.Object.{Commit, Tree}
alias ExGitObjectstore.Test.RepoHelper
@iterations 20
test "random DAGs round-trip through build → save → load" do
ex_unit_seed = ExUnit.configuration()[:seed]
for i <- 1..@iterations do
:rand.seed(:exsss, {ex_unit_seed, i, 0})
# Vary commit count so we exercise both tiny and non-trivial DAGs.
n_commits = :rand.uniform(40) + 5
merge_prob = :rand.uniform() * 0.3
repo = RepoHelper.memory_repo("rand-#{ex_unit_seed}-#{i}")
ExGitObjectstore.init(repo)
{commits, tips} = generate_dag(repo, n_commits, merge_prob)
register_tips(repo, tips)
{:ok, built} = Graph.build(repo)
assert Graph.size(built) == length(commits),
"iter=#{i}: expected #{length(commits)} commits in graph, got #{Graph.size(built)}"
verify_topology_invariants(built, commits)
# Round-trip through bytes.
bytes = BinaryFormat.serialize(built)
{:ok, roundtripped} = BinaryFormat.deserialize(bytes)
assert roundtripped.by_sha == built.by_sha, "iter=#{i}: round-trip mismatch"
assert Enum.sort(roundtripped.shas) == Enum.sort(built.shas)
# And through storage.
:ok = Graph.save(repo, built)
{:ok, loaded} = Graph.load(repo)
assert loaded.by_sha == built.by_sha, "iter=#{i}: load-from-storage mismatch"
end
end
# -- DAG generation --
#
# Start with one root. For each subsequent commit pick either a single
# existing commit as parent (plain child) or, with `merge_prob`, pick two
# distinct existing commits as parents (merge commit). Return the list of
# {sha, [parent_sha]} in creation order so we can later cross-check.
defp generate_dag(repo, n, merge_prob) do
{:ok, tree_sha} = Object.write(repo, Tree.new([]))
root_sha = write_commit(repo, tree_sha, [], timestamp(1))
commits = [{root_sha, []}]
{commits_rev, all_parents_used} =
Enum.reduce(2..n, {commits, MapSet.new()}, fn i, {acc, parents_used} ->
existing_shas = Enum.map(acc, fn {sha, _} -> sha end)
parents = pick_parents(existing_shas, merge_prob)
sha = write_commit(repo, tree_sha, parents, timestamp(i))
parents_used = Enum.reduce(parents, parents_used, &MapSet.put(&2, &1))
{[{sha, parents} | acc], parents_used}
end)
commits_created = Enum.reverse(commits_rev)
# Tips = commits no other commit points at.
all_shas = MapSet.new(commits_created, fn {sha, _} -> sha end)
tips = all_shas |> MapSet.difference(all_parents_used) |> MapSet.to_list()
{commits_created, tips}
end
defp pick_parents(existing, merge_prob) do
if length(existing) >= 2 and :rand.uniform() < merge_prob do
# Two distinct parents for a merge commit.
[a, b] = existing |> Enum.shuffle() |> Enum.take(2)
[a, b]
else
[Enum.random(existing)]
end
end
defp write_commit(repo, tree_sha, parents, ts) do
ident = "A <a@a.com> #{ts} +0000"
commit = %Commit{
tree: tree_sha,
parents: parents,
author: ident,
committer: ident,
message: "c\n"
}
{:ok, sha} = Object.write(repo, commit)
sha
end
defp register_tips(repo, tips) do
for {tip, idx} <- Enum.with_index(tips) do
:ok = ExGitObjectstore.create_branch(repo, "tip-#{idx}", tip)
end
end
defp timestamp(i), do: 1_700_000_000 + i
# -- Invariants --
defp verify_topology_invariants(graph, commits) do
expected_parents = Map.new(commits)
for {sha, parents} <- commits do
assert Graph.member?(graph, sha), "sha #{sha} missing from graph"
{:ok, got_parents} = Graph.parents(graph, sha)
assert got_parents == parents, "parent list mismatch for #{sha}"
{:ok, gen} = Graph.generation(graph, sha)
expected_gen = expected_generation(graph, parents)
assert gen == expected_gen,
"generation mismatch for #{sha}: expected #{expected_gen}, got #{gen}"
{:ok, ccd} = Graph.corrected_commit_date(graph, sha)
ctime = graph.by_sha[sha].commit_time
expected_ccd = expected_ccd(graph, parents, ctime)
assert ccd == expected_ccd,
"CCD mismatch for #{sha}: expected #{expected_ccd}, got #{ccd}"
end
# Sanity: every parent referenced by an entry must itself be in the graph.
for {_sha, %{parents: ps}} <- graph.by_sha, p <- ps do
assert Map.has_key?(expected_parents, p), "parent #{p} not in generated DAG"
assert Graph.member?(graph, p)
end
end
defp fetch_gen!(graph, sha) do
{:ok, g} = Graph.generation(graph, sha)
g
end
defp fetch_ccd!(graph, sha) do
{:ok, c} = Graph.corrected_commit_date(graph, sha)
c
end
defp expected_generation(_graph, []), do: 1
defp expected_generation(graph, parents),
do: 1 + Enum.max(Enum.map(parents, &fetch_gen!(graph, &1)))
defp expected_ccd(_graph, [], ctime), do: ctime
defp expected_ccd(graph, parents, ctime),
do: Enum.max([ctime | Enum.map(parents, &fetch_ccd!(graph, &1))])
end
▸
test/ex_git_objectstore/storage/filesystem_test.exs
+51
−0
@@ -255,4 +255,55 @@
assert {:ok, ^commit_sha} = ExGitObjectstore.resolve(repo, "HEAD")
end
end
describe "blob storage" do
test "put/get round-trips", %{repo: repo} do
data = :crypto.strong_rand_bytes(4096)
:ok = Repo.storage_call(repo, :put_blob, ["graph/commit-graph.v1", data])
assert {:ok, ^data} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "blob lives at <root>/<prefix>/blobs/<key>", %{repo: repo, root: root} do
:ok = Repo.storage_call(repo, :put_blob, ["graph/commit-graph.v1", "payload"])
assert File.read!(Path.join([root, "repos/test-repo/blobs/graph/commit-graph.v1"])) ==
"payload"
end
test "get on missing blob", %{repo: repo} do
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "put overwrites", %{repo: repo} do
:ok = Repo.storage_call(repo, :put_blob, ["k", "v1"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v2"])
assert {:ok, "v2"} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "delete removes", %{repo: repo} do
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
:ok = Repo.storage_call(repo, :delete_blob, ["k"])
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "delete on missing is :ok", %{repo: repo} do
assert :ok = Repo.storage_call(repo, :delete_blob, ["k"])
end
test "blob_exists?", %{repo: repo} do
refute Repo.storage_call(repo, :blob_exists?, ["k"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
assert Repo.storage_call(repo, :blob_exists?, ["k"])
end
test "rejects traversal", %{repo: repo} do
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :put_blob, ["../evil", "x"])
end
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :put_blob, ["/abs", "x"])
end
end
end
end
▸
test/ex_git_objectstore/storage/memory_test.exs
+70
−0
@@ -196,4 +196,74 @@
assert IO.iodata_to_binary(Enum.to_list(stream)) == pack_data
end
end
describe "blob storage" do
test "put and get blob round-trips" do
repo = RepoHelper.memory_repo()
data = :crypto.strong_rand_bytes(1024)
:ok = Repo.storage_call(repo, :put_blob, ["graph/commit-graph.v1", data])
assert {:ok, ^data} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "get non-existent blob returns :not_found" do
repo = RepoHelper.memory_repo()
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "put_blob overwrites existing blob" do
repo = RepoHelper.memory_repo()
:ok = Repo.storage_call(repo, :put_blob, ["k", "v1"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v2"])
assert {:ok, "v2"} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "blob_exists? reflects presence" do
repo = RepoHelper.memory_repo()
refute Repo.storage_call(repo, :blob_exists?, ["k"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
assert Repo.storage_call(repo, :blob_exists?, ["k"])
end
test "delete_blob removes the blob" do
repo = RepoHelper.memory_repo()
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
:ok = Repo.storage_call(repo, :delete_blob, ["k"])
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "delete_blob on missing key is :ok" do
repo = RepoHelper.memory_repo()
assert :ok = Repo.storage_call(repo, :delete_blob, ["k"])
end
test "blob keys are namespaced by repo prefix" do
repo_a = RepoHelper.memory_repo("repo-a")
# Reuse the same storage config by pointing another Repo at the same pid
{mod, cfg} = repo_a.storage
repo_b = Repo.new("repo-b", storage: {mod, cfg})
:ok = Repo.storage_call(repo_a, :put_blob, ["graph/commit-graph.v1", "a"])
:ok = Repo.storage_call(repo_b, :put_blob, ["graph/commit-graph.v1", "b"])
assert {:ok, "a"} = Repo.storage_call(repo_a, :get_blob, ["graph/commit-graph.v1"])
assert {:ok, "b"} = Repo.storage_call(repo_b, :get_blob, ["graph/commit-graph.v1"])
end
test "rejects path traversal" do
repo = RepoHelper.memory_repo()
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :put_blob, ["../evil", "x"])
end
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :put_blob, ["/abs", "x"])
end
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :get_blob, [""])
end
end
end
end
▸
test/ex_git_objectstore/storage/s3_test.exs
+36
−0
@@ -481,6 +481,42 @@
end
end
describe "blob storage" do
test "put/get round-trips", %{repo: repo} do
data = :crypto.strong_rand_bytes(2048)
:ok = Repo.storage_call(repo, :put_blob, ["graph/commit-graph.v1", data])
assert {:ok, ^data} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "get on missing blob", %{repo: repo} do
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["graph/commit-graph.v1"])
end
test "put overwrites", %{repo: repo} do
:ok = Repo.storage_call(repo, :put_blob, ["k", "v1"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v2"])
assert {:ok, "v2"} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "delete removes", %{repo: repo} do
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
:ok = Repo.storage_call(repo, :delete_blob, ["k"])
assert {:error, :not_found} = Repo.storage_call(repo, :get_blob, ["k"])
end
test "blob_exists?", %{repo: repo} do
refute Repo.storage_call(repo, :blob_exists?, ["k"])
:ok = Repo.storage_call(repo, :put_blob, ["k", "v"])
assert Repo.storage_call(repo, :blob_exists?, ["k"])
end
test "rejects traversal", %{repo: repo} do
assert_raise ArgumentError, fn ->
Repo.storage_call(repo, :put_blob, ["../evil", "x"])
end
end
end
defp list_all_keys(config, prefix, continuation_token, acc) do
opts =
[prefix: prefix] ++
▸
test/support/tracking_memory.ex
+24
−0
@@ -140,4 +140,28 @@
track(config, {:list_objects, prefix})
Memory.list_objects(mem_config(config), prefix)
end
@impl true
def get_blob(config, prefix, blob_key) do
track(config, {:get_blob, prefix, blob_key})
Memory.get_blob(mem_config(config), prefix, blob_key)
end
@impl true
def put_blob(config, prefix, blob_key, data) do
track(config, {:put_blob, prefix, blob_key})
Memory.put_blob(mem_config(config), prefix, blob_key, data)
end
@impl true
def delete_blob(config, prefix, blob_key) do
track(config, {:delete_blob, prefix, blob_key})
Memory.delete_blob(mem_config(config), prefix, blob_key)
end
@impl true
def blob_exists?(config, prefix, blob_key) do
track(config, {:blob_exists?, prefix, blob_key})
Memory.blob_exists?(mem_config(config), prefix, blob_key)
end
end