ref:988139cdd359b83d29ac5bfaaf08095237bdfae3

Phase 2: Storage backends + ref management

- Filesystem storage backend with S3-compatible key layout - S3 storage backend via ex_aws_s3 - packed-refs support for reading real git bare repos - Integration tests: create repos with git CLI, read through library - Filesystem tests: full CRUD for objects, refs, HEAD, packs - 83 tests passing Refs: notifd/anvil#62 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
SHA: 988139cdd359b83d29ac5bfaaf08095237bdfae3
Author: Cole Christensen <cole.christensen@macmillan.com>
Date: 2026-02-10 07:38
Parents: 2984a0d
4 files changed +915 -0
Type
lib/ex_git_objectstore/storage/filesystem.ex +303 −0
@@ -1,0 +1,303 @@
defmodule ExGitObjectstore.Storage.Filesystem do
@moduledoc """
Local filesystem storage backend.
Mirrors the S3 key layout on disk:
<root>/repos/<repo-id>/
HEAD
refs/heads/<branch>
refs/tags/<tag>
objects/<sha[0:2]>/<sha[2:]>
objects/pack/pack-<sha>.pack
objects/pack/pack-<sha>.idx
Config: `%{root: "/path/to/storage"}`
"""
@behaviour ExGitObjectstore.Storage
# -- Object operations --
@impl true
def get_object(config, prefix, sha) do
path = object_path(config, prefix, sha)
case File.read(path) do
{:ok, data} -> {:ok, data}
{:error, :enoent} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
@impl true
def put_object(config, prefix, sha, data) do
path = object_path(config, prefix, sha)
File.mkdir_p!(Path.dirname(path))
File.write!(path, data)
:ok
end
@impl true
def object_exists?(config, prefix, sha) do
File.exists?(object_path(config, prefix, sha))
end
# -- Pack operations --
@impl true
def list_packs(config, prefix) do
pack_dir = Path.join([config.root, prefix, "objects", "pack"])
case File.ls(pack_dir) do
{:ok, files} ->
packs =
files
|> Enum.filter(&String.ends_with?(&1, ".pack"))
|> Enum.map(fn file ->
file
|> String.replace_prefix("pack-", "")
|> String.replace_suffix(".pack", "")
end)
{:ok, packs}
{:error, :enoent} ->
{:ok, []}
{:error, reason} ->
{:error, reason}
end
end
@impl true
def get_pack(config, prefix, pack_sha) do
path = pack_path(config, prefix, pack_sha, "pack")
case File.read(path) do
{:ok, data} -> {:ok, data}
{:error, :enoent} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
@impl true
def get_pack_index(config, prefix, pack_sha) do
path = pack_path(config, prefix, pack_sha, "idx")
case File.read(path) do
{:ok, data} -> {:ok, data}
{:error, :enoent} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
@impl true
def put_pack(config, prefix, pack_sha, pack_data, idx_data) do
pack_p = pack_path(config, prefix, pack_sha, "pack")
idx_p = pack_path(config, prefix, pack_sha, "idx")
File.mkdir_p!(Path.dirname(pack_p))
File.write!(pack_p, pack_data)
File.write!(idx_p, idx_data)
:ok
end
@impl true
def stream_pack(config, prefix, pack_sha) do
path = pack_path(config, prefix, pack_sha, "pack")
if File.exists?(path) do
stream = File.stream!(path, 65_536)
{:ok, stream}
else
{:error, :not_found}
end
end
# -- Ref operations --
@impl true
def get_ref(config, prefix, ref_path) do
path = Path.join([config.root, prefix, ref_path])
case File.read(path) do
{:ok, data} -> {:ok, String.trim(data)}
{:error, :enoent} -> get_packed_ref(config, prefix, ref_path)
{:error, reason} -> {:error, reason}
end
end
@impl true
def put_ref(config, prefix, ref_path, new_sha, old_sha) do
path = Path.join([config.root, prefix, ref_path])
if old_sha do
# CAS: check current value
case File.read(path) do
{:ok, data} ->
if String.trim(data) == old_sha do
File.mkdir_p!(Path.dirname(path))
File.write!(path, new_sha <> "\n")
:ok
else
{:error, :cas_failed}
end
{:error, :enoent} ->
{:error, :cas_failed}
{:error, reason} ->
{:error, reason}
end
else
File.mkdir_p!(Path.dirname(path))
File.write!(path, new_sha <> "\n")
:ok
end
end
@impl true
def delete_ref(config, prefix, ref_path) do
path = Path.join([config.root, prefix, ref_path])
case File.rm(path) do
:ok -> :ok
{:error, :enoent} -> :ok
{:error, reason} -> {:error, reason}
end
end
@impl true
def list_refs(config, prefix, ref_prefix) do
base_dir = Path.join([config.root, prefix, ref_prefix])
# Start with loose refs
loose_refs =
case list_files_recursive(base_dir) do
{:ok, files} ->
Enum.map(files, fn file ->
ref_name = ref_prefix <> Path.relative_to(file, base_dir)
{:ok, sha} = File.read(file)
{ref_name, String.trim(sha)}
end)
{:error, _} ->
[]
end
# Merge with packed-refs (loose refs take precedence)
packed_refs = list_packed_refs(config, prefix, ref_prefix)
loose_names = MapSet.new(loose_refs, &elem(&1, 0))
merged =
packed_refs
|> Enum.reject(fn {name, _sha} -> MapSet.member?(loose_names, name) end)
|> Enum.concat(loose_refs)
|> Enum.sort()
{:ok, merged}
end
# -- HEAD --
@impl true
def get_head(config, prefix) do
path = Path.join([config.root, prefix, "HEAD"])
case File.read(path) do
{:ok, data} -> {:ok, String.trim(data)}
{:error, :enoent} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
@impl true
def put_head(config, prefix, target) do
path = Path.join([config.root, prefix, "HEAD"])
File.mkdir_p!(Path.dirname(path))
File.write!(path, target <> "\n")
:ok
end
# -- Private --
defp object_path(config, prefix, sha) do
<<dir::binary-size(2), rest::binary>> = sha
Path.join([config.root, prefix, "objects", dir, rest])
end
defp pack_path(config, prefix, pack_sha, ext) do
Path.join([config.root, prefix, "objects", "pack", "pack-#{pack_sha}.#{ext}"])
end
defp get_packed_ref(config, prefix, ref_path) do
packed_refs_path = Path.join([config.root, prefix, "packed-refs"])
case File.read(packed_refs_path) do
{:ok, content} ->
case parse_packed_refs(content) |> Map.get(ref_path) do
nil -> {:error, :not_found}
sha -> {:ok, sha}
end
{:error, :enoent} ->
{:error, :not_found}
{:error, reason} ->
{:error, reason}
end
end
defp list_packed_refs(config, prefix, ref_prefix) do
packed_refs_path = Path.join([config.root, prefix, "packed-refs"])
case File.read(packed_refs_path) do
{:ok, content} ->
parse_packed_refs(content)
|> Enum.filter(fn {name, _sha} -> String.starts_with?(name, ref_prefix) end)
{:error, _} ->
[]
end
end
defp parse_packed_refs(content) do
content
|> String.split("\n")
|> Enum.reject(fn line ->
line == "" or String.starts_with?(line, "#") or String.starts_with?(line, "^")
end)
|> Enum.map(fn line ->
case String.split(line, " ", parts: 2) do
[sha, ref_name] -> {ref_name, sha}
_ -> nil
end
end)
|> Enum.reject(&is_nil/1)
|> Map.new()
end
defp list_files_recursive(dir) do
case File.ls(dir) do
{:ok, entries} ->
files =
Enum.flat_map(entries, fn entry ->
path = Path.join(dir, entry)
if File.dir?(path) do
case list_files_recursive(path) do
{:ok, sub_files} -> sub_files
_ -> []
end
else
[path]
end
end)
{:ok, files}
{:error, reason} ->
{:error, reason}
end
end
end
lib/ex_git_objectstore/storage/s3.ex +260 −0
@@ -1,0 +1,260 @@
defmodule ExGitObjectstore.Storage.S3 do
@moduledoc """
S3-compatible storage backend (AWS S3, MinIO, etc.).
Config:
%{
bucket: "my-bucket",
ex_aws_config: [ # optional, passed to ExAws operations
access_key_id: "...",
secret_access_key: "...",
region: "us-east-1",
host: "localhost", # for MinIO
port: 9000,
scheme: "http://"
]
}
S3 key layout:
<prefix>/HEAD
<prefix>/refs/heads/<branch>
<prefix>/objects/<sha[0:2]>/<sha[2:]>
<prefix>/objects/pack/pack-<sha>.pack
<prefix>/objects/pack/pack-<sha>.idx
"""
@behaviour ExGitObjectstore.Storage
# -- Object operations --
@impl true
def get_object(config, prefix, sha) do
key = object_key(prefix, sha)
s3_get(config, key)
end
@impl true
def put_object(config, prefix, sha, data) do
key = object_key(prefix, sha)
s3_put(config, key, data)
end
@impl true
def object_exists?(config, prefix, sha) do
key = object_key(prefix, sha)
case s3_head(config, key) do
:ok -> true
{:error, _} -> false
end
end
# -- Pack operations --
@impl true
def list_packs(config, prefix) do
pack_prefix = "#{prefix}/objects/pack/pack-"
case s3_list(config, pack_prefix) do
{:ok, keys} ->
packs =
keys
|> Enum.filter(&String.ends_with?(&1, ".pack"))
|> Enum.map(fn key ->
key
|> String.replace_prefix(pack_prefix, "")
|> String.replace_suffix(".pack", "")
end)
{:ok, packs}
{:error, _} = err ->
err
end
end
@impl true
def get_pack(config, prefix, pack_sha) do
key = pack_key(prefix, pack_sha, "pack")
s3_get(config, key)
end
@impl true
def get_pack_index(config, prefix, pack_sha) do
key = pack_key(prefix, pack_sha, "idx")
s3_get(config, key)
end
@impl true
def put_pack(config, prefix, pack_sha, pack_data, idx_data) do
with :ok <- s3_put(config, pack_key(prefix, pack_sha, "pack"), pack_data),
:ok <- s3_put(config, pack_key(prefix, pack_sha, "idx"), idx_data) do
:ok
end
end
@impl true
def stream_pack(config, prefix, pack_sha) do
# For S3, we download the full pack and wrap in a list.
# For large packs, a chunked Range-request approach would be better.
case get_pack(config, prefix, pack_sha) do
{:ok, data} -> {:ok, [data]}
{:error, _} = err -> err
end
end
# -- Ref operations --
@impl true
def get_ref(config, prefix, ref_path) do
key = "#{prefix}/#{ref_path}"
case s3_get(config, key) do
{:ok, data} -> {:ok, String.trim(data)}
{:error, _} = err -> err
end
end
@impl true
def put_ref(config, prefix, ref_path, new_sha, old_sha) do
key = "#{prefix}/#{ref_path}"
if old_sha do
# CAS: read current value first
case s3_get(config, key) do
{:ok, data} ->
if String.trim(data) == old_sha do
s3_put(config, key, new_sha <> "\n")
else
{:error, :cas_failed}
end
{:error, :not_found} ->
{:error, :cas_failed}
{:error, _} = err ->
err
end
else
s3_put(config, key, new_sha <> "\n")
end
end
@impl true
def delete_ref(config, prefix, ref_path) do
key = "#{prefix}/#{ref_path}"
s3_delete(config, key)
end
@impl true
def list_refs(config, prefix, ref_prefix) do
full_prefix = "#{prefix}/#{ref_prefix}"
case s3_list(config, full_prefix) do
{:ok, keys} ->
refs =
keys
|> Enum.map(fn key ->
ref_name = String.replace_prefix(key, "#{prefix}/", "")
case s3_get(config, key) do
{:ok, data} -> {ref_name, String.trim(data)}
_ -> nil
end
end)
|> Enum.reject(&is_nil/1)
|> Enum.sort()
{:ok, refs}
{:error, _} = err ->
err
end
end
# -- HEAD --
@impl true
def get_head(config, prefix) do
key = "#{prefix}/HEAD"
case s3_get(config, key) do
{:ok, data} -> {:ok, String.trim(data)}
{:error, _} = err -> err
end
end
@impl true
def put_head(config, prefix, target) do
key = "#{prefix}/HEAD"
s3_put(config, key, target <> "\n")
end
# -- Private S3 helpers --
defp object_key(prefix, sha) do
<<dir::binary-size(2), rest::binary>> = sha
"#{prefix}/objects/#{dir}/#{rest}"
end
defp pack_key(prefix, pack_sha, ext) do
"#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}"
end
defp s3_get(config, key) do
op = ExAws.S3.get_object(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do
{:ok, %{body: body}} -> {:ok, body}
{:error, {:http_error, 404, _}} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
defp s3_put(config, key, data) do
op = ExAws.S3.put_object(config.bucket, key, data)
case ExAws.request(op, ex_aws_config(config)) do
{:ok, _} -> :ok
{:error, reason} -> {:error, reason}
end
end
defp s3_delete(config, key) do
op = ExAws.S3.delete_object(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do
{:ok, _} -> :ok
{:error, reason} -> {:error, reason}
end
end
defp s3_head(config, key) do
op = ExAws.S3.head_object(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do
{:ok, _} -> :ok
{:error, {:http_error, 404, _}} -> {:error, :not_found}
{:error, reason} -> {:error, reason}
end
end
defp s3_list(config, prefix) do
op = ExAws.S3.list_objects_v2(config.bucket, prefix: prefix)
case ExAws.request(op, ex_aws_config(config)) do
{:ok, %{body: %{contents: contents}}} ->
keys = Enum.map(contents, & &1.key)
{:ok, keys}
{:ok, %{body: _}} ->
{:ok, []}
{:error, reason} ->
{:error, reason}
end
end
defp ex_aws_config(%{ex_aws_config: config}), do: config
defp ex_aws_config(_), do: []
end