@@ -1,0 +1,359 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.Maintenance do
@moduledoc """
Server-side repack — the `git gc` equivalent for objectstore repositories.
`receive_pack` explodes every pushed packfile into individual loose objects,
and nothing ever put them back together, so repositories accumulate loose
objects without bound. A production repository observed while investigating
this held **236,756 loose objects across 39 repositories with 4 packs total**;
the largest single repository was 1.1 GB spread over 134,921 files.
That is a latency problem before it is a disk problem. `ObjectResolver` reads
packs first and falls back to loose, so with everything loose every read is a
separate `File.read` on the filesystem backend — and a separate HTTP GET on
the S3 backend.
## What `repack/2` does
1. Walks every ref to collect the set of reachable object SHAs.
2. Lists the loose objects.
3. Packs `loose ∩ reachable`, in batches, into new packfiles.
4. Verifies each object reads back out of the pack.
5. Only then deletes the loose copies.
Objects that are already in a pack are left where they are — this is an
incremental repack (`git repack` without `-a`), not a full rewrite.
## What it deliberately does not do
**It never deletes an unreachable object.** Pruning needs a grace period,
because a push in flight can have written objects that no ref points at yet;
git's default is two weeks. Unreachable loose objects are counted and
reported so a future pruning pass has a number to work from, and otherwise
left alone. Deleting them here would be the one mistake in this module that
loses data permanently.
## Ordering is a correctness property, not a style choice
A reader resolves packs first, then loose. So the pack must exist and be
readable *before* the loose copy goes away:
write pack -> verify readable -> delete loose
Any other order leaves a window where an object exists nowhere. If the
process dies mid-repack, the worst case is loose objects that are also in a
pack — wasted disk, no data loss — and the next run cleans them up.
## Memory
Repacking a 1.1 GB repository must not need 1.1 GB of heap; the production
host has 3 GB total. Object *contents* are therefore never accumulated across
a batch boundary: `:max_batch_bytes` caps how much raw object content one
packfile pass holds (default 64 MB). What does span the whole run is one SHA
string per object — for 236,756 objects that is tens of megabytes, which is
the price of knowing what is reachable.
"""
require Logger
alias ExGitObjectstore.Object.{Blob, Commit, Tag, Tree}
alias ExGitObjectstore.{ObjectResolver, Repo}
alias ExGitObjectstore.Pack.Writer
@default_max_batch_bytes 64 * 1024 * 1024
@default_max_batch_objects 50_000
# `git gc --auto` fires at 6700 loose objects. The same number is used here
# for the same reason: it is large enough that ordinary pushes don't trigger
# constant repacking, small enough that lookups stay fast.
@default_loose_threshold 6_700
# Many small packs are their own problem — every lookup miss scans another
# index. git's `gc.autoPackLimit` default is 50.
@default_pack_threshold 50
@type stats :: %{
loose_before: non_neg_integer(),
reachable: non_neg_integer(),
packed: non_neg_integer(),
deleted: non_neg_integer(),
unreachable_kept: non_neg_integer(),
packs_written: [String.t()]
}
@doc """
Counts loose objects and packs without changing anything.
"""
@spec stats(Repo.t()) :: {:ok, %{loose: non_neg_integer(), packs: non_neg_integer()}}
def stats(%Repo{} = repo) do
{:ok, %{loose: length(list_loose(repo)), packs: length(list_packs(repo))}}
end
@doc """
Whether this repository is worth repacking right now — the `git gc --auto`
check.
True when loose objects exceed `:loose_threshold` (default #{@default_loose_threshold},
matching git's `gc.auto`) or packs exceed `:pack_threshold` (default
#{@default_pack_threshold}, matching git's `gc.autoPackLimit`).
Cheap enough to call after a push: it lists directory entries, it does not
read objects.
"""
@spec needs_repack?(Repo.t(), keyword()) :: boolean()
def needs_repack?(%Repo{} = repo, opts \\ []) do
loose_threshold = Keyword.get(opts, :loose_threshold, @default_loose_threshold)
pack_threshold = Keyword.get(opts, :pack_threshold, @default_pack_threshold)
length(list_loose(repo)) > loose_threshold or
length(list_packs(repo)) > pack_threshold
end
@doc """
Pack loose reachable objects and remove the loose copies.
## Options
* `:max_batch_bytes` — cap on raw object content held per packfile pass
(default #{@default_max_batch_bytes}). Bounds peak memory.
* `:max_batch_objects` — cap on objects per packfile pass
(default #{@default_max_batch_objects}).
* `:dry_run` — compute and report, write nothing (default `false`).
Returns `{:ok, stats}`. Safe to run repeatedly; a second run over an
already-packed repository packs nothing.
"""
@spec repack(Repo.t(), keyword()) :: {:ok, stats()} | {:error, term()}
def repack(%Repo{} = repo, opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run, false)
loose = repo |> list_loose() |> Enum.uniq()
with {:ok, reachable} <- reachable_shas(repo) do
to_pack = loose |> Enum.filter(&Map.has_key?(reachable, &1)) |> Enum.sort()
unreachable_kept = length(loose) - length(to_pack)
base = %{
loose_before: length(loose),
reachable: map_size(reachable),
packed: 0,
deleted: 0,
unreachable_kept: unreachable_kept,
packs_written: []
}
cond do
to_pack == [] ->
{:ok, base}
dry_run? ->
{:ok, %{base | packed: length(to_pack)}}
true ->
run_batches(repo, to_pack, base, opts)
end
end
end
# ── Batched packing ──────────────────────────────────────────────────
defp run_batches(repo, to_pack, stats, opts) do
max_bytes = Keyword.get(opts, :max_batch_bytes, @default_max_batch_bytes)
max_objects = Keyword.get(opts, :max_batch_objects, @default_max_batch_objects)
to_pack
|> batch_by_size(repo, max_bytes, max_objects)
|> Enum.reduce_while({:ok, stats}, fn batch, {:ok, acc} ->
case pack_batch(repo, batch) do
{:ok, pack_sha, entries} ->
deleted = delete_packed(repo, entries)
{:cont,
{:ok,
%{
acc
| packed: acc.packed + length(entries),
deleted: acc.deleted + deleted,
packs_written: acc.packs_written ++ [pack_sha]
}}}
{:error, _} = err ->
{:halt, err}
end
end)
end
# Groups SHAs into batches whose combined raw content stays under the byte
# cap. Sizes come from the objects themselves, so this reads each object
# once here and once when packing — deliberate: holding every object's bytes
# to avoid the second read is exactly the memory blow-up being avoided.
defp batch_by_size(shas, repo, max_bytes, max_objects) do
shas
|> Enum.chunk_while(
{[], 0, 0},
fn sha, {batch, bytes, count} ->
size = object_size(repo, sha)
if batch != [] and (bytes + size > max_bytes or count + 1 > max_objects) do
{:cont, Enum.reverse(batch), {[sha], size, 1}}
else
{:cont, {[sha | batch], bytes + size, count + 1}}
end
end,
fn
{[], _, _} -> {:cont, []}
{batch, _, _} -> {:cont, Enum.reverse(batch), {[], 0, 0}}
end
)
|> Enum.reject(&(&1 == []))
end
defp object_size(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %Blob{content: c}} -> byte_size(c)
{:ok, _other} -> 4_096
_ -> 0
end
end
defp pack_batch(repo, shas) do
entries =
shas
|> Enum.map(&pack_entry(repo, &1))
|> Enum.reject(&is_nil/1)
if entries == [] do
{:ok, nil, []}
else
{pack_data, idx_data, pack_sha} = Writer.generate_with_index(entries)
case Repo.storage_call(repo, :put_pack, [pack_sha, pack_data, idx_data]) do
:ok -> verify_pack(repo, pack_sha, entries)
{:error, _} = err -> err
end
end
end
defp pack_entry(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{} = c} -> {:commit, Commit.encode_content(c), sha}
{:ok, %Tree{} = t} -> {:tree, Tree.encode_content(t), sha}
{:ok, %Blob{content: content}} -> {:blob, content, sha}
{:ok, %Tag{} = t} -> {:tag, Tag.encode_content(t), sha}
_ -> nil
end
end
# Every object must be readable from the new pack before any loose copy is
# removed. The per-process pack caches in `ObjectResolver` are cleared first,
# otherwise this would re-read the pre-repack view and verify nothing.
defp verify_pack(repo, pack_sha, entries) do
ObjectResolver.clear_pack_cache()
missing =
Enum.reject(entries, fn {_type, _content, sha} ->
match?({:ok, _}, ObjectResolver.read(repo, sha))
end)
if missing == [] do
{:ok, pack_sha, entries}
else
shas = Enum.map(missing, fn {_, _, sha} -> sha end)
Logger.error(
"Maintenance.repack: pack #{pack_sha} written but #{length(shas)} objects " <>
"did not read back; keeping all loose copies. First: #{inspect(Enum.take(shas, 5))}"
)
{:error, {:verify_failed, shas}}
end
end
defp delete_packed(repo, entries) do
Enum.count(entries, fn {_type, _content, sha} ->
Repo.storage_call(repo, :delete_object, [sha]) == :ok
end)
end
# ── Reachability ─────────────────────────────────────────────────────
# Iterative worklist, not recursion: a deep commit chain would otherwise be
# bounded by stack rather than by the visited set. Only SHAs are retained.
# A plain map, not a MapSet: the set is only ever probed by key, and a map
# keeps dialyzer's opaque-type checking out of a hot recursive call.
@spec reachable_shas(Repo.t()) :: {:ok, %{optional(String.t()) => true}}
defp reachable_shas(repo) do
roots = ref_targets(repo)
{:ok, walk(repo, roots, %{})}
end
@spec walk(Repo.t(), [String.t()], %{optional(String.t()) => true}) ::
%{optional(String.t()) => true}
defp walk(_repo, [], visited), do: visited
defp walk(repo, [sha | rest], visited) do
if Map.has_key?(visited, sha) do
walk(repo, rest, visited)
else
walk(repo, children(repo, sha) ++ rest, Map.put(visited, sha, true))
end
end
defp children(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{tree: tree, parents: parents}} -> [tree | parents]
{:ok, %Tree{entries: entries}} -> Enum.map(entries, & &1.sha)
{:ok, %Tag{object: object}} -> [object]
_ -> []
end
end
defp ref_targets(repo) do
case Repo.storage_call(repo, :list_refs, ["refs/"]) do
{:ok, refs} ->
refs
|> Enum.map(fn
{_name, sha} -> sha
sha when is_binary(sha) -> sha
end)
|> Enum.filter(&valid_sha?/1)
|> Enum.uniq()
_ ->
[]
end
end
defp valid_sha?(sha) when is_binary(sha), do: byte_size(sha) == 40
defp valid_sha?(_), do: false
# ── Storage listings ─────────────────────────────────────────────────
defp list_loose(repo) do
case Repo.storage_call(repo, :list_objects, []) do
{:ok, shas} -> shas
_ -> []
end
end
defp list_packs(repo) do
case Repo.storage_call(repo, :list_packs, []) do
{:ok, packs} -> packs
_ -> []
end
end
end