ref:e9f91f101dbc6363991624f53d56f6b419e1550b

Merge pull request #4 from notifd/fix/code-review-critical-high

Fix critical and high severity code review findings
SHA: e9f91f101dbc6363991624f53d56f6b419e1550b
Author: Cole Christensen <cole.christensen+github@gmail.com>
Date: 2026-02-11 02:27
27 files changed +1706 -247
Type
lib/ex_git_objectstore/diff.ex +2 −7
@@ -86,13 +86,8 @@
{:ok, changes} <- diff_trees(repo, old_commit.tree, new_commit.tree) do
file_diffs =
Enum.map(changes, fn change ->
case diff_blobs(repo, change.old_sha, change.new_sha, opts) do
{:ok, hunks} ->
%{path: change.path, status: change.status, hunks: hunks}
{:error, _} ->
%{path: change.path, status: change.status, hunks: []}
end
{:ok, hunks} = diff_blobs(repo, change.old_sha, change.new_sha, opts)
%{path: change.path, status: change.status, hunks: hunks}
end)
{:ok, file_diffs}
lib/ex_git_objectstore/object.ex +51 −16
@@ -23,5 +23,8 @@
alias ExGitObjectstore.Object.{Blob, Commit, Tree, Tag}
alias ExGitObjectstore.Repo
# Maximum decompressed object size (128 MB)
@max_object_size 128 * 1024 * 1024
@type object_type :: :blob | :commit | :tree | :tag
@type t :: Blob.t() | Commit.t() | Tree.t() | Tag.t()
@@ -47,22 +50,23 @@
@doc """
Encode a git object to its stored format (zlib compressed).
Raises on compression failure (extremely rare).
Returns `{:ok, compressed}` or `{:error, reason}`.
"""
@spec encode(t()) :: binary()
@spec encode(t()) :: {:ok, binary()} | {:error, term()}
def encode(object) do
case zlib_compress(encode_raw(object)) do
zlib_compress(encode_raw(object))
{:ok, compressed} -> compressed
{:error, reason} -> raise "zlib compression failed: #{inspect(reason)}"
end
end
@doc """
Safe version of encode/1 — returns `{:ok, compressed}` or `{:error, reason}`.
Encode a git object to its stored format (zlib compressed).
Raises on compression failure.
"""
@spec safe_encode(t()) :: {:ok, binary()} | {:error, term()}
def safe_encode(object) do
@spec encode!(t()) :: binary()
def encode!(object) do
case encode(object) do
{:ok, compressed} -> compressed
{:error, reason} -> raise "zlib compression failed: #{inspect(reason)}"
zlib_compress(encode_raw(object))
end
end
@doc """
@@ -82,8 +86,13 @@
@spec decode_raw(binary()) :: {:ok, t()} | {:error, term()}
def decode_raw(raw) do
case parse_header(raw) do
{:ok, type, size, content} ->
# Validate that header size matches actual content length
{:ok, type, _size, content} ->
decode_typed(type, content)
if byte_size(content) != size do
{:error, {:size_mismatch, expected: size, actual: byte_size(content)}}
else
decode_typed(type, content)
end
{:error, _} = err ->
err
@@ -96,7 +105,23 @@
@spec read(Repo.t(), String.t()) :: {:ok, t()} | {:error, term()}
def read(%Repo{} = repo, sha) do
case Repo.storage_call(repo, :get_object, [sha]) do
{:ok, data} ->
# Verify SHA-1 integrity: decompress, hash, and compare to requested SHA
case safe_decompress(data) do
{:ok, raw} ->
actual_sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
if actual_sha != sha do
{:error, {:sha_mismatch, expected: sha, actual: actual_sha}}
else
decode_raw(raw)
end
{:error, _} = err ->
err
end
{:error, _} = err ->
err
{:ok, data} -> decode(data)
{:error, _} = err -> err
end
end
@@ -108,7 +133,7 @@
def write(%Repo{} = repo, object) do
sha = hash(object)
case safe_encode(object) do
case encode(object) do
{:ok, compressed} ->
case Repo.storage_call(repo, :put_object, [sha, compressed]) do
:ok -> {:ok, sha}
@@ -186,13 +211,23 @@
end
defp safe_decompress(data) do
safe_decompress(data, @max_object_size)
end
defp safe_decompress(data, max_size) do
z = :zlib.open()
try do
:zlib.inflateInit(z)
result = :zlib.inflate(z, data)
:zlib.inflateEnd(z)
{:ok, IO.iodata_to_binary(result)}
decompressed = IO.iodata_to_binary(result)
if byte_size(decompressed) > max_size do
{:error, {:object_too_large, byte_size(decompressed), max_size}}
else
{:ok, decompressed}
end
rescue
e -> {:error, {:decompress_failed, Exception.message(e)}}
after
lib/ex_git_objectstore/object/commit.ex +29 −11
@@ -35,6 +35,7 @@
extra_headers: [{String.t(), String.t()}]
}
@enforce_keys [:tree, :author, :committer, :message]
defstruct [
:tree,
:author,
@@ -59,7 +60,7 @@
lines =
if commit.gpgsig do
lines ++ [encode_multiline_header("gpgsig", commit.gpgsig)]
lines ++ ["gpgsig #{commit.gpgsig}\n"]
else
lines
end
@@ -67,7 +68,7 @@
lines =
lines ++
Enum.map(commit.extra_headers, fn {key, value} ->
"#{key} #{value}\n"
encode_multiline_header(key, value)
end)
lines = lines ++ ["\n", commit.message]
@@ -101,6 +102,23 @@
end)
end
# Encode a header value that may contain newlines as continuation lines.
# Per the git spec, continuation lines are prefixed with a space character.
defp encode_multiline_header(key, value) do
if String.contains?(value, "\n") do
# Split on newlines and prefix continuation lines with space
[first | rest] = String.split(value, "\n")
lines =
["#{key} #{first}\n"] ++
Enum.map(rest, fn line -> " #{line}\n" end)
IO.iodata_to_binary(lines)
else
"#{key} #{value}\n"
end
end
# Handle GPG signature continuation lines (start with space)
defp fold_continuation_lines(lines) do
lines
@@ -118,27 +136,27 @@
end
defp build_commit(headers, message) do
fields =
commit =
Enum.reduce(headers, %__MODULE__{message: message}, fn
Enum.reduce(headers, %{message: message, parents: [], extra_headers: []}, fn
{"tree", sha}, acc ->
Map.put(acc, :tree, sha)
%{acc | tree: sha}
{"parent", sha}, acc ->
%{acc | parents: acc.parents ++ [sha]}
Map.update(acc, :parents, [sha], &(&1 ++ [sha]))
{"author", value}, acc ->
%{acc | author: value}
Map.put(acc, :author, value)
{"committer", value}, acc ->
Map.put(acc, :committer, value)
%{acc | committer: value}
{"gpgsig", value}, acc ->
%{acc | gpgsig: value}
Map.put(acc, :gpgsig, value)
{key, value}, acc ->
Map.update(acc, :extra_headers, [{key, value}], &(&1 ++ [{key, value}]))
%{acc | extra_headers: acc.extra_headers ++ [{key, value}]}
end)
{:ok, commit}
{:ok, struct!(__MODULE__, fields)}
end
end
lib/ex_git_objectstore/object/tag.ex +60 −17
@@ -29,25 +29,42 @@
object: String.t(),
type: String.t(),
tag: String.t(),
tagger: String.t(),
message: String.t()
tagger: String.t() | nil,
message: String.t(),
extra_headers: [{String.t(), String.t()}]
}
# tagger is optional — older git versions created tags without it
@enforce_keys [:object, :type, :tag, :message]
defstruct [:object, :type, :tag, :tagger, :message, extra_headers: []]
defstruct [:object, :type, :tag, :tagger, :message]
@doc """
Encode tag to its content format.
"""
@spec encode_content(t()) :: binary()
def encode_content(%__MODULE__{} = tag) do
IO.iodata_to_binary([
lines = [
"object #{tag.object}\n",
"type #{tag.type}\n",
"tag #{tag.tag}\n"
]
lines =
if tag.tagger do
lines ++ ["tagger #{tag.tagger}\n"]
else
lines
end
lines =
lines ++
Enum.map(tag.extra_headers || [], fn {key, value} ->
"#{key} #{value}\n"
end)
lines = lines ++ ["\n", tag.message]
IO.iodata_to_binary(lines)
"tag #{tag.tag}\n",
"tagger #{tag.tagger}\n",
"\n",
tag.message
])
end
@doc """
@@ -60,6 +77,7 @@
headers =
headers_str
|> String.split("\n")
|> fold_continuation_lines()
|> Enum.map(fn line ->
case String.split(line, " ", parts: 2) do
[key, value] -> {key, value}
@@ -67,19 +85,44 @@
end
end)
fields =
Enum.reduce(headers, %{message: message, extra_headers: []}, fn
{"object", sha}, acc ->
tag =
Enum.reduce(headers, %__MODULE__{message: message}, fn
{"object", sha}, acc -> %{acc | object: sha}
{"type", type}, acc -> %{acc | type: type}
{"tag", name}, acc -> %{acc | tag: name}
{"tagger", value}, acc -> %{acc | tagger: value}
_, acc -> acc
Map.put(acc, :object, sha)
{"type", type}, acc ->
Map.put(acc, :type, type)
{"tag", name}, acc ->
Map.put(acc, :tag, name)
{"tagger", value}, acc ->
Map.put(acc, :tagger, value)
{key, value}, acc ->
Map.update(acc, :extra_headers, [{key, value}], &(&1 ++ [{key, value}]))
end)
{:ok, struct!(__MODULE__, fields)}
{:ok, tag}
_ ->
{:error, :invalid_tag_format}
end
end
# Handle continuation lines (start with space) for multiline headers
defp fold_continuation_lines(lines) do
lines
|> Enum.reduce([], fn line, acc ->
if String.starts_with?(line, " ") do
case acc do
[prev | rest] -> [prev <> "\n" <> line | rest]
[] -> [line]
end
else
[line | acc]
end
end)
|> Enum.reverse()
end
end
lib/ex_git_objectstore/object/tree.ex +46 −3
@@ -28,13 +28,53 @@
@doc """
Create a tree from a list of entries.
Entries are sorted according to git's tree sorting rules.
Modes are canonicalized to their standard git representations.
"""
@spec new([entry()]) :: t()
def new(entries) do
entries = Enum.map(entries, &canonicalize_entry/1)
# Validate entry names
Enum.each(entries, fn entry ->
case validate_entry_name(entry.name) do
:ok -> :ok
{:error, reason} -> raise ArgumentError, "invalid tree entry name: #{inspect(reason)}"
end
end)
%__MODULE__{entries: sort_entries(entries)}
end
defp canonicalize_entry(%{mode: mode} = entry) do
%{entry | mode: canonicalize_mode(mode)}
end
@doc """
Canonicalize a git tree entry mode to its standard representation.
"""
@spec canonicalize_mode(String.t()) :: String.t()
def canonicalize_mode("644"), do: "100644"
def canonicalize_mode("755"), do: "100755"
def canonicalize_mode("0644"), do: "100644"
def canonicalize_mode("0755"), do: "100755"
def canonicalize_mode("0100644"), do: "100644"
def canonicalize_mode("0100755"), do: "100755"
def canonicalize_mode("040000"), do: "40000"
def canonicalize_mode("0040000"), do: "40000"
# Gitlink (submodule) mode
def canonicalize_mode("0160000"), do: "160000"
def canonicalize_mode(mode), do: mode
@doc """
Returns true if the given mode represents a tree (directory).
"""
@spec directory_mode?(String.t()) :: boolean()
def directory_mode?("40000"), do: true
def directory_mode?("040000"), do: true
def directory_mode?("0040000"), do: true
def directory_mode?(_), do: false
@doc """
Validate a tree entry name according to git rules.
Returns `:ok` or `{:error, reason}`.
"""
@@ -44,6 +84,7 @@
name == "" -> {:error, {:invalid_entry_name, name, :empty}}
name == "." -> {:error, {:invalid_entry_name, name, :dot}}
name == ".." -> {:error, {:invalid_entry_name, name, :dotdot}}
String.downcase(name) == ".git" -> {:error, {:invalid_entry_name, name, :dotgit}}
String.contains?(name, "/") -> {:error, {:invalid_entry_name, name, :slash}}
String.contains?(name, <<0>>) -> {:error, {:invalid_entry_name, name, :null_byte}}
true -> :ok
@@ -98,11 +139,13 @@
end
# Git sorts tree entries by name, treating directories as if they have a trailing "/".
# This matches git's base_name_compare() in tree-diff.c.
defp sort_entries(entries) do
Enum.sort_by(entries, fn entry ->
case entry.mode do
"40000" -> entry.name <> "/"
_ -> entry.name
if directory_mode?(entry.mode) do
entry.name <> "/"
else
entry.name
end
end)
end
lib/ex_git_objectstore/pack/delta.ex +12 −3
@@ -49,6 +49,8 @@
{:error, {:size_mismatch, expected: target_size, got: byte_size(result_bin)}}
end
end
catch
{:error, _} = err -> err
end
# Read a variable-length integer (used for base/target sizes in delta header)
@@ -131,12 +133,19 @@
end
# Read one byte from data if the corresponding bit is set in cmd
defp read_if_bit(cmd, bit, data) do
defp read_if_bit(cmd, bit, <<byte, rest::binary>> = _data) do
if Bitwise.band(cmd, Bitwise.bsl(1, bit)) != 0 do
<<byte, rest::binary>> = data
{byte, rest}
else
{0, <<byte, rest::binary>>}
end
end
{0, data}
defp read_if_bit(cmd, bit, <<>>) do
if Bitwise.band(cmd, Bitwise.bsl(1, bit)) != 0 do
throw({:error, :truncated_delta_copy})
else
{0, <<>>}
end
end
end
lib/ex_git_objectstore/pack/index.ex +51 −17
@@ -46,27 +46,61 @@
@max_pack_objects 10_000_000
@spec parse(binary()) :: {:ok, t()} | {:error, term()}
def parse(<<@idx_magic, @idx_version, rest::binary>>) do
with {:ok, fanout, rest} <- parse_fanout(rest),
count = elem(fanout, 255),
:ok <- validate_fanout(fanout),
:ok <- validate_count(count),
{:ok, shas, rest} <- parse_shas(rest, count),
{:ok, _crcs, rest} <- parse_crcs(rest, count),
{:ok, offsets_4, rest} <- parse_offsets_4(rest, count),
{:ok, large_offsets} <- parse_large_offsets(rest, offsets_4),
{:ok, offset_map} <- build_offset_map(shas, offsets_4, large_offsets) do
{:ok,
%__MODULE__{
fanout: fanout,
shas: shas,
offsets: offset_map,
count: count
}}
def parse(<<@idx_magic, @idx_version, _rest::binary>> = idx_data) do
# Verify index checksum first (PK4)
with :ok <- verify_index_checksum(idx_data) do
<<_magic_version::binary-size(8), rest::binary>> = idx_data
with {:ok, fanout, rest} <- parse_fanout(rest),
count = elem(fanout, 255),
:ok <- validate_fanout(fanout),
:ok <- validate_count(count),
{:ok, shas, rest} <- parse_shas(rest, count),
:ok <- validate_shas_sorted(shas),
{:ok, _crcs, rest} <- parse_crcs(rest, count),
{:ok, offsets_4, rest} <- parse_offsets_4(rest, count),
{:ok, large_offsets} <- parse_large_offsets(rest, offsets_4),
{:ok, offset_map} <- build_offset_map(shas, offsets_4, large_offsets) do
{:ok,
%__MODULE__{
fanout: fanout,
shas: shas,
offsets: offset_map,
count: count
}}
end
end
end
def parse(_), do: {:error, :invalid_idx_header}
# Verify the trailing 20-byte index checksum
defp verify_index_checksum(idx_data) when byte_size(idx_data) >= 28 do
body_len = byte_size(idx_data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = idx_data
expected = :crypto.hash(:sha, body)
if checksum == expected do
:ok
else
{:error, :index_checksum_mismatch}
end
end
defp verify_index_checksum(_), do: {:error, :index_too_short}
# Validate that SHA list is sorted (PK7)
defp validate_shas_sorted([]), do: :ok
defp validate_shas_sorted([_]), do: :ok
defp validate_shas_sorted(shas) do
sorted? =
shas
|> Enum.chunk_every(2, 1, :discard)
|> Enum.all?(fn [a, b] -> a < b end)
if sorted?, do: :ok, else: {:error, :shas_not_sorted}
end
@doc """
Look up the pack offset for a given SHA.
lib/ex_git_objectstore/pack/reader.ex +165 −22
@@ -32,6 +32,7 @@
@pack_signature "PACK"
@max_pack_objects 10_000_000
@max_delta_depth 50
# Object types (3-bit)
@obj_commit 1
@@ -52,14 +53,30 @@
Returns a list of `%{type: atom, data: binary, offset: integer}`.
"""
@spec parse(binary()) :: {:ok, [pack_object()]} | {:error, term()}
def parse(<<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, rest::binary>>)
when version == 2 do
def parse(
<<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, _rest::binary>> =
pack_data
)
when version in [2, 3] do
if count > @max_pack_objects do
{:error, {:pack_too_large, count}}
else
# Verify trailing checksum (PK1)
case verify_pack_checksum(pack_data) do
:ok ->
# Strip the trailing 20-byte checksum before parsing entries
entries_len = byte_size(pack_data) - 12 - 20
<<_header::binary-size(12), entries_data::binary-size(entries_len),
_checksum::binary-size(20)>> = pack_data
case parse_entries(entries_data, count, [], %{}) do
{:ok, entries, _cache} -> {:ok, entries}
{:error, _} = err -> err
end
{:error, _} = err ->
err
case parse_entries(rest, count, [], %{}) do
{:ok, entries, _cache} -> {:ok, entries}
{:error, _} = err -> err
end
end
end
@@ -67,6 +84,24 @@
def parse(_), do: {:error, :invalid_pack_header}
@doc """
Verify the trailing SHA-1 checksum of a pack file.
"""
@spec verify_pack_checksum(binary()) :: :ok | {:error, term()}
def verify_pack_checksum(pack_data) when byte_size(pack_data) >= 32 do
body_len = byte_size(pack_data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = pack_data
expected = :crypto.hash(:sha, body)
if checksum == expected do
:ok
else
{:error, :pack_checksum_mismatch}
end
end
def verify_pack_checksum(_), do: {:error, :pack_too_short}
@doc """
Read a single object at the given offset from a packfile binary.
Resolves deltas recursively using the pack data itself.
"""
@@ -81,13 +116,7 @@
@spec read_object(binary(), non_neg_integer(), map()) ::
{:ok, {atom(), binary()}} | {:error, term()}
def read_object(pack_data, offset, cache) do
case Map.get(cache, offset) do
{type, data} ->
{:ok, {type, data}}
nil ->
do_read_object(pack_data, offset, cache)
end
do_read_object(pack_data, offset, cache, 0)
end
# -- Private --
@@ -169,7 +198,21 @@
end
end
defp do_read_object(pack_data, offset, cache) do
defp do_read_object(_pack_data, _offset, _cache, depth) when depth > @max_delta_depth do
{:error, :max_delta_depth_exceeded}
end
defp do_read_object(pack_data, offset, cache, depth) do
case Map.get(cache, offset) do
{type, data} ->
{:ok, {type, data}}
nil ->
resolve_object(pack_data, offset, cache, depth)
end
end
defp resolve_object(pack_data, offset, cache, depth) do
<<_skip::binary-size(offset), data::binary>> = pack_data
case parse_object_header(data) do
@@ -197,7 +240,7 @@
with {:ok, delta_data, _} <- decompress_data(compressed),
{:ok, {base_type, base_data}} <-
do_read_object(pack_data, base_offset, cache, depth + 1),
read_object(pack_data, base_offset, cache),
{:ok, result} <- Delta.apply(base_data, delta_data) do
{:ok, {base_type, result}}
end
@@ -215,7 +258,7 @@
with {:ok, delta_data, _} <- decompress_data(compressed),
{:ok, base_offset} <- find_sha_offset(pack_data, base_sha, cache),
{:ok, {base_type, base_data}} <-
read_object(pack_data, base_offset, cache),
do_read_object(pack_data, base_offset, cache, depth + 1),
{:ok, result} <- Delta.apply(base_data, delta_data) do
{:ok, {base_type, result}}
end
@@ -352,12 +395,112 @@
defp type_num_to_atom(@obj_blob), do: :blob
defp type_num_to_atom(@obj_tag), do: :tag
# Build an in-memory SHA-to-offset index by scanning pack entries.
# For non-delta objects, compute the SHA from type+content.
# This enables REF_DELTA resolution without a .idx file.
defp find_sha_offset(pack_data, sha, cache) do
# Check cache first (caller may have provided from .idx)
case Map.get(cache, {:sha, sha}) do
offset when is_integer(offset) ->
{:ok, offset}
nil ->
# Build index by scanning the pack
case build_sha_index(pack_data) do
{:ok, index} ->
case Map.get(index, sha) do
nil -> {:error, {:ref_delta_base_not_found, sha}}
offset -> {:ok, offset}
end
{:error, _} = err ->
err
end
# TODO: Implement REF_DELTA resolution by building an in-memory index from
# the pack data. Currently REF_DELTA objects can only be resolved when the
# caller provides offsets via the cache (e.g., from a .idx file lookup).
# Without this, thin packs from git-fetch that use REF_DELTA will fail.
# See: https://git-scm.com/docs/pack-format#_deltified_representation
defp find_sha_offset(_pack_data, _sha, _cache) do
{:error, :ref_delta_not_supported}
end
end
@doc """
Build a SHA-to-offset index by scanning all non-delta entries in a pack.
"""
@spec build_sha_index(binary()) :: {:ok, map()} | {:error, term()}
def build_sha_index(
<<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, rest::binary>>
)
when version in [2, 3] do
scan_entries_for_index(rest, count, 12, %{})
end
def build_sha_index(_), do: {:error, :invalid_pack_header}
defp scan_entries_for_index(_data, 0, _offset, index), do: {:ok, index}
defp scan_entries_for_index(data, remaining, offset, index) do
case parse_object_header(data) do
{:ok, type_num, _size, header_len, rest_after_header} ->
case type_num do
t when t in [@obj_commit, @obj_tree, @obj_blob, @obj_tag] ->
type_str = Atom.to_string(type_num_to_atom(t))
case decompress_data(rest_after_header) do
{:ok, content, compressed_len} ->
# Compute SHA: hash("type size\0content")
raw = "#{type_str} #{byte_size(content)}\0" <> content
sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
consumed = header_len + compressed_len
<<_::binary-size(consumed), next_data::binary>> = data
scan_entries_for_index(
next_data,
remaining - 1,
offset + consumed,
Map.put(index, sha, offset)
)
{:error, _} = err ->
err
end
@obj_ofs_delta ->
case parse_ofs_delta_offset(rest_after_header) do
{:ok, _neg_offset, ofs_len, rest_after_ofs} ->
case decompress_data(rest_after_ofs) do
{:ok, _delta_data, compressed_len} ->
consumed = header_len + ofs_len + compressed_len
<<_::binary-size(consumed), next_data::binary>> = data
# We don't index delta objects (they need resolution first)
scan_entries_for_index(next_data, remaining - 1, offset + consumed, index)
{:error, _} = err ->
err
end
{:error, _} = err ->
err
end
@obj_ref_delta ->
if byte_size(rest_after_header) >= 20 do
<<_base_sha::binary-size(20), rest_after_sha::binary>> = rest_after_header
case decompress_data(rest_after_sha) do
{:ok, _delta_data, compressed_len} ->
consumed = header_len + 20 + compressed_len
<<_::binary-size(consumed), next_data::binary>> = data
scan_entries_for_index(next_data, remaining - 1, offset + consumed, index)
{:error, _} = err ->
err
end
else
{:error, :truncated_ref_delta}
end
_ ->
{:error, {:unknown_object_type, type_num}}
end
{:error, _} = err ->
err
end
end
end
lib/ex_git_objectstore/pack/writer.ex +14 −10
@@ -180,19 +180,23 @@
end
defp build_fanout(sorted_entries) do
# 256 entries, each is cumulative count of objects with first SHA byte <= N
sha_first_bytes =
Enum.map(sorted_entries, fn {sha, _offset, _crc} ->
# O(N) fanout computation: count objects per bucket, then cumulative sum
counts = :array.new(256, default: 0)
counts =
Enum.reduce(sorted_entries, counts, fn {sha, _offset, _crc}, acc ->
<<first_byte, _rest::binary>> = elem(Base.decode16(sha, case: :mixed), 1)
first_byte
:array.set(first_byte, :array.get(first_byte, acc) + 1, acc)
end)
fanout =
for i <- 0..255 do
Enum.count(sha_first_bytes, fn b -> b <= i end)
end
# Build cumulative fanout
{fanout_list, _} =
Enum.reduce(0..255, {[], 0}, fn i, {acc, cumulative} ->
total = cumulative + :array.get(i, counts)
{[<<total::unsigned-big-32>> | acc], total}
end)
fanout
|> Enum.map(fn count -> <<count::unsigned-big-32>> end)
fanout_list
|> Enum.reverse()
|> IO.iodata_to_binary()
end
lib/ex_git_objectstore/protocol/pkt_line.ex +30 −5
@@ -23,14 +23,18 @@
- `NNNN<data>` — data packet where NNNN is total length (including the 4 prefix bytes)
Minimum data packet length is `0005` (4 prefix + 1 data byte).
Maximum is `ffff` (65535 bytes total, 65531 data bytes).
Maximum is `fff0` (65520 bytes total, 65516 data bytes).
"""
@flush "0000"
@delim "0001"
@response_end "0002"
# Max total pkt-line is 65520 (0xFFF0) bytes per spec.
# Max payload = 65520 - 4 (length prefix) = 65516
@max_pkt_len 65520
@max_data_len 65520
# For sideband: max payload (65516) - 1 band byte = 65515
@max_sideband_data 65515
@doc """
Encode a single data line as a pkt-line.
@@ -40,6 +44,11 @@
def encode(data) when is_binary(data) do
data = if String.ends_with?(data, "\n"), do: data, else: data <> "\n"
len = byte_size(data) + 4
if len > @max_pkt_len do
raise ArgumentError, "pkt-line too long: #{len} bytes (max #{@max_pkt_len})"
end
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0")
<<hex_len::binary, data::binary>>
end
@@ -50,6 +59,11 @@
@spec encode_raw(binary()) :: binary()
def encode_raw(data) when is_binary(data) do
len = byte_size(data) + 4
if len > @max_pkt_len do
raise ArgumentError, "pkt-line too long: #{len} bytes (max #{@max_pkt_len})"
end
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0")
<<hex_len::binary, data::binary>>
end
@@ -109,8 +123,8 @@
{len, ""} when len >= 4 ->
if byte_size(data) >= len do
<<_hex::binary-size(4), payload::binary-size(len - 4), rest::binary>> = data
# Strip trailing newline if present
# Strip exactly one trailing LF if present (spec says receivers should strip it)
payload = strip_one_trailing_lf(payload)
payload = String.trim_trailing(payload, "\n")
{:ok, {:data, payload}, rest}
else
{:need_more, data}
@@ -143,7 +157,7 @@
"""
@spec encode_sideband(1 | 2 | 3, binary()) :: iodata()
def encode_sideband(band, data) when band in [1, 2, 3] do
chunks = chunk_binary(data, @max_sideband_data)
chunks = chunk_binary(data, @max_data_len)
Enum.map(chunks, fn chunk ->
encode_raw(<<band, chunk::binary>>)
@@ -167,5 +181,16 @@
{:error, _} = err ->
err
end
end
defp strip_one_trailing_lf(<<>>), do: <<>>
defp strip_one_trailing_lf(bin) do
size = byte_size(bin) - 1
case bin do
<<prefix::binary-size(size), "\n">> -> prefix
_ -> bin
end
end
lib/ex_git_objectstore/protocol/receive_pack.ex +145 −46
@@ -27,12 +27,15 @@
in and reads response data out.
"""
alias ExGitObjectstore.{Object, Ref, Repo}
alias ExGitObjectstore.{Object, ObjectResolver, Ref, Repo}
alias ExGitObjectstore.Pack.Reader
alias ExGitObjectstore.Protocol.PktLine
@zero_sha String.duplicate("0", 40)
# Maximum pack buffer size (256 MB)
@max_pack_size 256 * 1024 * 1024
# Capabilities we actually implement. Others are scaffolded but not yet functional:
# TODO: ofs-delta — Writer doesn't generate OFS_DELTA yet
# TODO: side-band-64k — not yet used in report responses
@@ -48,14 +51,16 @@
repo: Repo.t(),
phase: :advertise | :commands | :pack | :done,
commands: [command()],
client_caps: MapSet.t(),
pack_buffer: binary(),
result: :ok | {:error, term()}
result: nil | :ok | {:error, term()}
}
defstruct [
:repo,
phase: :advertise,
commands: [],
client_caps: MapSet.new(),
pack_buffer: <<>>,
result: nil
]
@@ -81,7 +86,14 @@
@spec feed(state(), binary()) :: {binary(), state()}
def feed(%__MODULE__{phase: :commands} = state, data) do
case parse_commands(data) do
{:ok, commands, rest} ->
state = %{state | commands: commands, phase: :pack, pack_buffer: rest}
{:ok, commands, client_caps, rest} ->
state = %{
state
| commands: commands,
client_caps: client_caps,
phase: :pack,
pack_buffer: rest
}
# Check if the pack is already complete in the buffer
maybe_process_pack(state)
@@ -97,8 +109,15 @@
end
def feed(%__MODULE__{phase: :pack} = state, data) do
state = %{state | pack_buffer: state.pack_buffer <> data}
maybe_process_pack(state)
new_buffer = state.pack_buffer <> data
if byte_size(new_buffer) > @max_pack_size do
report = build_error_report(:pack_too_large, state.commands)
{report, %{state | phase: :done, result: {:error, :pack_too_large}}}
else
state = %{state | pack_buffer: new_buffer}
maybe_process_pack(state)
end
end
def feed(%__MODULE__{phase: :done} = state, _data) do
@@ -115,20 +134,20 @@
# -- Private --
defp build_advertisement(repo) do
refs = list_all_refs_with_head(repo)
refs = list_all_refs(repo)
case refs do
[] ->
# Empty repo — advertise capabilities with zero SHA
line = "#{@zero_sha} capabilities^{}\0 #{@capabilities}"
line = "#{@zero_sha} capabilities^{}\0#{@capabilities}"
PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] ->
first_line = "#{first_sha} #{first_ref}\0 #{@capabilities}"
first_line = "#{first_sha} #{first_ref}\0#{@capabilities}"
lines =
[PktLine.encode(first_line)] ++
Enum.map(rest, fn {ref, sha} ->
PktLine.encode("#{sha} #{ref}")
Enum.flat_map(rest, fn {ref, sha} ->
[PktLine.encode("#{sha} #{ref}")]
end) ++
[PktLine.flush()]
@@ -137,7 +156,9 @@
end
end
defp list_all_refs(repo) do
# Build the ref list with HEAD first (spec: HEAD MUST appear first).
# Also includes peeled tag refs (^{}) for annotated tags.
defp list_all_refs_with_head(repo) do
heads =
case Ref.list(repo, "refs/heads/") do
{:ok, refs} -> refs
@@ -150,28 +171,76 @@
_ -> []
end
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
sorted =
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
# Add peeled tag refs
sorted_with_peeled =
Enum.flat_map(sorted, fn {ref, sha} = entry ->
if String.starts_with?(ref, "refs/tags/") do
case peel_tag(repo, sha) do
{:ok, peeled_sha} when peeled_sha != sha ->
[entry, {"#{ref}^{}", peeled_sha}]
_ ->
[entry]
end
else
[entry]
end
end)
# Resolve HEAD and prepend it
case resolve_head_sha(repo) do
{:ok, head_sha} ->
[{"HEAD", head_sha} | sorted_with_peeled]
_ ->
sorted_with_peeled
end
end
defp resolve_head_sha(repo) do
case Ref.get_head(repo) do
{:ok, "ref: " <> ref_path} -> Ref.get(repo, ref_path)
{:ok, sha} -> {:ok, sha}
error -> error
end
end
defp peel_tag(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %ExGitObjectstore.Object.Tag{object: target_sha}} ->
{:ok, target_sha}
_ ->
{:ok, sha}
end
end
defp parse_commands(data) do
parse_command_lines(data, [])
parse_command_lines(data, [], nil)
end
defp parse_command_lines(data, acc, caps) do
defp parse_command_lines(data, acc) do
case PktLine.decode_one(data) do
{:ok, :flush, rest} ->
client_caps = caps || MapSet.new()
if acc == [] do
# No commands — client is just probing
{:ok, [], rest}
{:ok, [], client_caps, rest}
else
{:ok, Enum.reverse(acc), rest}
{:ok, Enum.reverse(acc), client_caps, rest}
end
{:ok, {:data, line}, rest} ->
case parse_command_line(line) do
{:ok, cmd} ->
parse_command_lines(rest, [cmd | acc])
{:ok, cmd, line_caps} ->
# Only the first command line carries capabilities
caps = caps || line_caps
parse_command_lines(rest, [cmd | acc], caps)
{:error, _} = err ->
err
@@ -187,18 +256,26 @@
defp parse_command_line(line) do
# Format: "<old-sha> <new-sha> <ref-name>[\0<capabilities>]"
# Strip capabilities if present
line =
# Extract capabilities if present
{cmd_str, caps} =
case String.split(line, "\0", parts: 2) do
[cmd, _caps] -> cmd
[cmd] -> cmd
[cmd, caps_str] ->
cap_set =
caps_str
|> String.split(" ", trim: true)
|> MapSet.new()
{cmd, cap_set}
[cmd] ->
{cmd, MapSet.new()}
end
case String.split(line, " ", parts: 3) do
case String.split(cmd_str, " ", parts: 3) do
[old_sha, new_sha, ref] when byte_size(old_sha) == 40 and byte_size(new_sha) == 40 ->
{:ok, %{ref: ref, old_sha: old_sha, new_sha: new_sha}, caps}
{:ok, %{ref: ref, old_sha: old_sha, new_sha: new_sha}}
_ ->
{:error, {:invalid_command, line}}
{:error, {:invalid_command, cmd_str}}
end
end
@@ -217,10 +294,6 @@
:incomplete ->
{<<>>, state}
{:error, reason} ->
report = build_error_report(reason)
{report, %{state | phase: :done, result: {:error, reason}}}
end
end
end
@@ -231,7 +304,7 @@
# A complete pack has: header (12 bytes) + entries + 20-byte checksum
# We can't easily know the total size without parsing all entries,
# so we verify the trailing SHA-1 checksum
if byte_size(data) > 32 do
if byte_size(data) >= 32 do
body_len = byte_size(data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = data
expected = :crypto.hash(:sha, body)
@@ -256,7 +329,7 @@
process_ref_updates(state)
{:error, reason} ->
report = build_error_report(reason)
report = build_error_report(reason, state.commands)
{report, %{state | phase: :done, result: {:error, reason}}}
end
end
@@ -267,10 +340,15 @@
Enum.reduce_while(entries, :ok, fn entry, :ok ->
raw = Object.encode_raw_from_type(entry.type, entry.data)
sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
case zlib_compress(raw) do
{:ok, compressed} ->
case Repo.storage_call(repo, :put_object, [sha, compressed]) do
:ok -> {:cont, :ok}
compressed = zlib_compress(raw)
{:error, _} = err -> {:halt, err}
end
case Repo.storage_call(repo, :put_object, [sha, compressed]) do
:ok -> {:cont, :ok}
{:error, _} = err -> {:halt, err}
{:error, _} = err ->
{:halt, err}
end
end)
@@ -287,8 +365,17 @@
{cmd.ref, result}
end)
report = build_report(results)
{report, %{state | phase: :done, result: :ok}}
# Track whether any ref updates failed
any_errors? = Enum.any?(results, fn {_ref, r} -> match?({:error, _}, r) end)
overall_result = if any_errors?, do: {:error, :some_refs_failed}, else: :ok
# Only send report-status if client requested it (or if no commands/caps parsed)
if MapSet.member?(state.client_caps, "report-status") or MapSet.size(state.client_caps) == 0 do
report = build_report(results)
{report, %{state | phase: :done, result: overall_result}}
else
{<<>>, %{state | phase: :done, result: overall_result}}
end
end
defp apply_ref_command(repo, %{old_sha: @zero_sha, new_sha: new_sha, ref: ref}) do
@@ -314,21 +401,33 @@
PktLine.encode("ok #{ref}")
{ref, {:error, reason}} ->
PktLine.encode("ng #{ref} #{inspect(reason)}")
PktLine.encode("ng #{ref} #{sanitize_error(reason)}")
end) ++
[PktLine.flush()]
IO.iodata_to_binary(lines)
end
defp build_error_report(reason, commands \\ []) do
defp build_error_report(reason) do
error_msg = sanitize_error(reason)
ref_lines =
Enum.map(commands, fn cmd ->
PktLine.encode("ng #{cmd.ref} #{error_msg}")
end)
lines =
[PktLine.encode("unpack #{error_msg}")] ++
ref_lines ++
[PktLine.flush()]
lines = [
PktLine.encode("unpack #{inspect(reason)}"),
PktLine.flush()
]
IO.iodata_to_binary(lines)
end
defp sanitize_error(reason) when is_atom(reason), do: Atom.to_string(reason)
defp sanitize_error(reason) when is_binary(reason), do: reason
defp sanitize_error({tag, _details}) when is_atom(tag), do: Atom.to_string(tag)
defp sanitize_error(_), do: "internal_error"
defp zlib_compress(data) do
z = :zlib.open()
@@ -337,7 +436,7 @@
:zlib.deflateInit(z)
compressed = :zlib.deflate(z, data, :finish)
:zlib.deflateEnd(z)
IO.iodata_to_binary(compressed)
{:ok, IO.iodata_to_binary(compressed)}
rescue
e -> {:error, {:compress_failed, Exception.message(e)}}
after
lib/ex_git_objectstore/protocol/upload_pack.ex +192 −59
@@ -75,11 +75,20 @@
generate_pack_response(state)
{:ok, wants, haves, :continue} ->
# Client wants more negotiation rounds — simplified: just ACK and continue
state = %{state | wants: wants ++ state.wants, haves: haves ++ state.haves}
nak = PktLine.encode("NAK")
{nak, state}
# Find common objects: ACK the last have that we have in our repo
case find_common(state.repo, haves) do
{:ok, common_sha} ->
ack = PktLine.encode("ACK #{common_sha}")
state = %{state | common: [common_sha | state.common]}
{ack, state}
:none ->
nak = PktLine.encode("NAK")
{nak, state}
end
{:error, _reason} ->
nak = PktLine.encode("NAK")
{nak, %{state | phase: :done}}
@@ -100,21 +109,21 @@
# -- Private --
defp build_advertisement(repo) do
refs = list_all_refs(repo)
refs = list_all_refs_with_head(repo)
symref = head_symref(repo)
caps = build_capabilities(symref)
case refs do
[] ->
line = "#{@zero_sha} capabilities^{}\0 #{caps}"
line = "#{@zero_sha} capabilities^{}\0#{caps}"
PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] ->
first_line = "#{first_sha} #{first_ref}\0 #{caps}"
first_line = "#{first_sha} #{first_ref}\0#{caps}"
lines =
[PktLine.encode(first_line)] ++
Enum.map(rest, fn {ref, sha} ->
PktLine.encode("#{sha} #{ref}")
Enum.flat_map(rest, fn {ref, sha} ->
[PktLine.encode("#{sha} #{ref}")]
end) ++
[PktLine.flush()]
@@ -133,7 +142,9 @@
defp build_capabilities(nil), do: @capabilities
defp build_capabilities(ref), do: "#{@capabilities} symref=HEAD:#{ref}"
defp list_all_refs(repo) do
# Build the ref list with HEAD first (spec: HEAD MUST appear first).
# Also includes peeled tag refs (^{}) for annotated tags.
defp list_all_refs_with_head(repo) do
heads =
case Ref.list(repo, "refs/heads/") do
{:ok, refs} -> refs
@@ -146,43 +157,140 @@
_ -> []
end
sorted =
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
# Add peeled tag refs: for annotated tags, add a refname^{} entry
# showing the commit the tag points to
sorted_with_peeled =
Enum.flat_map(sorted, fn {ref, sha} = entry ->
if String.starts_with?(ref, "refs/tags/") do
case peel_tag(repo, sha) do
{:ok, peeled_sha} when peeled_sha != sha ->
[entry, {"#{ref}^{}", peeled_sha}]
_ ->
[entry]
end
else
[entry]
end
end)
# Resolve HEAD and prepend it
case resolve_head_sha(repo) do
{:ok, head_sha} ->
[{"HEAD", head_sha} | sorted_with_peeled]
_ ->
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
sorted_with_peeled
end
end
defp resolve_head_sha(repo) do
case Ref.get_head(repo) do
{:ok, "ref: " <> ref_path} -> Ref.get(repo, ref_path)
{:ok, sha} -> {:ok, sha}
error -> error
end
end
# Peel an annotated tag to its target object SHA.
defp peel_tag(repo, sha) do
case ObjectResolver.read(repo, sha) do
{:ok, %ExGitObjectstore.Object.Tag{object: target_sha}} ->
{:ok, target_sha}
_ ->
{:ok, sha}
end
end
defp parse_wants_haves(data) do
# Phase 1: Parse want lines until flush
case parse_want_lines(data, []) do
{:ok, wants, rest} ->
# Phase 2: Parse have lines until "done"
case parse_have_lines(rest, []) do
{:ok, haves, :done} ->
{:ok, wants, haves, :done}
{:ok, haves, :continue} ->
{:ok, wants, haves, :continue}
{:error, _} = err ->
err
end
{:error, _} = err ->
parse_wh_lines(data, [], [], nil)
err
end
end
defp parse_wh_lines(data, wants, haves, _terminator) do
defp parse_want_lines(data, acc) do
case PktLine.decode_one(data) do
{:ok, :flush, rest} ->
{:ok, Enum.reverse(acc), rest}
{:ok, {:data, "want " <> sha_and_caps}, rest} ->
sha = sha_and_caps |> String.split(" ", parts: 2) |> List.first() |> String.trim()
parse_want_lines(rest, [sha | acc])
{:ok, {:data, "done"}, _rest} ->
# Client sent done without any haves (simple clone)
{:ok, Enum.reverse(acc), <<>>}
{:ok, {:data, _other}, rest} ->
parse_want_lines(rest, acc)
{:need_more, _} ->
{:ok, Enum.reverse(acc), <<>>}
{:error, _} = err ->
err
end
end
defp parse_have_lines(<<>>, acc) do
# No have data — this is a clone (wants only, no haves)
{:ok, Enum.reverse(acc), :done}
end
{:ok, :flush, _rest} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :done}
defp parse_have_lines(data, acc) do
case PktLine.decode_one(data) do
{:ok, {:data, "done"}, _rest} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :done}
{:ok, Enum.reverse(acc), :done}
{:ok, {:data, "want " <> sha_and_caps}, rest} ->
# First want line may have capabilities after SHA
sha = sha_and_caps |> String.split(" ", parts: 2) |> List.first() |> String.trim()
parse_wh_lines(rest, [sha | wants], haves, nil)
{:ok, :flush, _rest} ->
{:ok, Enum.reverse(acc), :continue}
{:ok, {:data, "have " <> sha}, rest} ->
sha = String.trim(sha)
parse_wh_lines(rest, wants, [sha | haves], nil)
parse_have_lines(rest, [sha | acc])
{:ok, {:data, _other}, rest} ->
# Skip unknown lines
parse_wh_lines(rest, wants, haves, nil)
parse_have_lines(rest, acc)
{:need_more, _} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :continue}
{:ok, Enum.reverse(acc), :continue}
{:error, _} = err ->
err
end
end
# Check if any of the given SHAs exist in our repo.
# Returns {:ok, sha} for the last one found, or :none.
defp find_common(repo, haves) do
Enum.reduce(haves, :none, fn sha, acc ->
case ObjectResolver.read(repo, sha) do
{:ok, _} -> {:ok, sha}
_ -> acc
end
end)
end
defp generate_pack_response(state) do
# Collect all objects needed for the wants, excluding objects reachable from haves
case collect_objects(state.repo, state.wants, state.haves) do
@@ -202,22 +310,28 @@
{response, %{state | phase: :done}}
{:error, reason} ->
nak = PktLine.encode("ERR #{inspect(reason)}")
nak = PktLine.encode("ERR #{sanitize_error(reason)}")
{nak, %{state | phase: :done}}
end
end
# Defensive: handle various error shapes even though current callers only pass binary
@dialyzer {:nowarn_function, sanitize_error: 1}
defp sanitize_error(reason) when is_binary(reason), do: reason
defp sanitize_error(reason) when is_atom(reason), do: Atom.to_string(reason)
defp sanitize_error({tag, _details}) when is_atom(tag), do: Atom.to_string(tag)
defp sanitize_error(_), do: "internal_error"
defp collect_objects(repo, wants, haves) do
# Walk from each want SHA and collect all reachable objects
have_set = MapSet.new(haves)
try do
objects =
wants
|> Enum.flat_map(fn sha ->
collect_reachable(repo, sha, have_set)
{objects, _visited} =
Enum.reduce(wants, {[], MapSet.new()}, fn sha, {acc, visited} ->
{new_objects, visited} = collect_reachable(repo, sha, have_set, visited)
{acc ++ new_objects, visited}
end)
|> Enum.uniq_by(fn {_type, _data, sha} -> sha end)
{:ok, objects}
rescue
@@ -225,64 +339,83 @@
end
end
defp collect_reachable(repo, sha, exclude_set) do
if MapSet.member?(exclude_set, sha) do
[]
defp collect_reachable(repo, sha, exclude_set, visited) do
if MapSet.member?(exclude_set, sha) or MapSet.member?(visited, sha) do
{[], visited}
else
visited = MapSet.put(visited, sha)
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{} = commit} ->
# Collect the tree and all referenced objects
tree_objects = collect_tree_objects(repo, commit.tree)
{tree_objects, visited} = collect_tree_objects(repo, commit.tree, visited)
# Recurse into parents (but stop at haves)
{parent_objects, visited} =
Enum.reduce(commit.parents, {[], visited}, fn parent_sha, {acc, vis} ->
parent_objects =
{objs, vis} = collect_reachable(repo, parent_sha, exclude_set, vis)
{acc ++ objs, vis}
Enum.flat_map(commit.parents, fn parent_sha ->
collect_reachable(repo, parent_sha, exclude_set)
end)
[{:commit, Object.encode_content_only(commit), sha} | tree_objects ++ parent_objects]
objects = [
{:commit, Object.encode_content_only(commit), sha} | tree_objects ++ parent_objects
]
{objects, visited}
{:ok, %Tree{} = tree} ->
collect_tree_entry_objects(repo, tree, sha)
collect_tree_entry_objects(repo, tree, sha, visited)
{:ok, %Blob{content: content}} ->
[{:blob, content, sha}]
{[{:blob, content, sha}], visited}
{:error, _} ->
[]
{[], visited}
end
end
end
defp collect_tree_objects(repo, tree_sha, visited) do
if MapSet.member?(visited, tree_sha) do
defp collect_tree_objects(repo, tree_sha) do
case ObjectResolver.read(repo, tree_sha) do
{:ok, %Tree{} = tree} ->
{[], visited}
else
case ObjectResolver.read(repo, tree_sha) do
{:ok, %Tree{} = tree} ->
collect_tree_entry_objects(repo, tree, tree_sha, visited)
collect_tree_entry_objects(repo, tree, tree_sha)
_ ->
{[], visited}
end
_ ->
[]
end
end
defp collect_tree_entry_objects(repo, %Tree{} = tree, tree_sha) do
defp collect_tree_entry_objects(repo, %Tree{} = tree, tree_sha, visited) do
visited = MapSet.put(visited, tree_sha)
tree_content = Tree.encode_content(tree)
entry = [{:tree, tree_content, tree_sha}]
child_objects =
Enum.flat_map(tree.entries, fn
%{mode: "40000", sha: sha} ->
{child_objects, visited} =
Enum.reduce(tree.entries, {[], visited}, fn
%{mode: "40000", sha: sha}, {acc, vis} ->
{objs, vis} = collect_tree_objects(repo, sha, vis)
{acc ++ objs, vis}
%{sha: sha}, {acc, vis} ->
if MapSet.member?(vis, sha) do
{acc, vis}
else
vis = MapSet.put(vis, sha)
collect_tree_objects(repo, sha)
%{sha: sha} ->
case ObjectResolver.read(repo, sha) do
{:ok, %Blob{content: content}} ->
[{:blob, content, sha}]
case ObjectResolver.read(repo, sha) do
{:ok, %Blob{content: content}} ->
{acc ++ [{:blob, content, sha}], vis}
_ ->
_ ->
{acc, vis}
end
[]
end
end)
{entry ++ child_objects, visited}
entry ++ child_objects
end
end
lib/ex_git_objectstore/ref.ex +30 −3
@@ -24,7 +24,9 @@
"""
@spec get(Repo.t(), String.t()) :: {:ok, String.t()} | {:error, term()}
def get(%Repo{} = repo, ref_path) do
with :ok <- validate_ref_name(ref_path) do
Repo.storage_call(repo, :get_ref, [ref_path])
end
Repo.storage_call(repo, :get_ref, [ref_path])
end
@doc """
@@ -44,7 +46,9 @@
"""
@spec delete(Repo.t(), String.t()) :: :ok | {:error, term()}
def delete(%Repo{} = repo, ref_path) do
with :ok <- validate_ref_name(ref_path) do
Repo.storage_call(repo, :delete_ref, [ref_path])
Repo.storage_call(repo, :delete_ref, [ref_path])
end
end
@doc """
@@ -52,7 +56,9 @@
"""
@spec list(Repo.t(), String.t()) :: {:ok, [{String.t(), String.t()}]} | {:error, term()}
def list(%Repo{} = repo, ref_prefix) do
with :ok <- validate_ref_prefix(ref_prefix) do
Repo.storage_call(repo, :list_refs, [ref_prefix])
Repo.storage_call(repo, :list_refs, [ref_prefix])
end
end
@doc """
@@ -157,6 +163,27 @@
String.starts_with?(ref_path, "/") ->
{:error, {:invalid_ref_name, ref_path, :leading_slash}}
true ->
:ok
end
end
@doc """
Validate a ref prefix for listing operations.
Rejects path traversal and null bytes but allows trailing slashes.
"""
@spec validate_ref_prefix(String.t()) :: :ok | {:error, term()}
def validate_ref_prefix(prefix) do
cond do
String.contains?(prefix, "..") ->
{:error, {:invalid_ref_prefix, prefix, :dotdot}}
String.contains?(prefix, <<0>>) ->
{:error, {:invalid_ref_prefix, prefix, :null_byte}}
String.starts_with?(prefix, "/") ->
{:error, {:invalid_ref_prefix, prefix, :leading_slash}}
true ->
:ok
lib/ex_git_objectstore/storage/filesystem.ex +6 −6
@@ -133,7 +133,7 @@
@impl true
def get_ref(config, prefix, ref_path) do
path = Path.join([config.root, prefix, ref_path])
path = safe_path(config.root, Path.join([prefix, ref_path]))
case File.read(path) do
{:ok, data} -> {:ok, String.trim(data)}
@@ -144,6 +144,6 @@
@impl true
def put_ref(config, prefix, ref_path, new_sha, old_sha) do
path = Path.join([config.root, prefix, ref_path])
path = safe_path(config.root, Path.join([prefix, ref_path]))
lock_path = path <> ".lock"
File.mkdir_p!(Path.dirname(path))
@@ -195,7 +195,7 @@
@impl true
def delete_ref(config, prefix, ref_path) do
path = safe_path(config.root, Path.join([prefix, ref_path]))
path = Path.join([config.root, prefix, ref_path])
case File.rm(path) do
:ok -> :ok
@@ -206,7 +206,7 @@
@impl true
def list_refs(config, prefix, ref_prefix) do
base_dir = Path.join([config.root, prefix, ref_prefix])
base_dir = safe_path(config.root, Path.join([prefix, ref_prefix]))
# Start with loose refs
loose_refs =
@@ -239,7 +239,7 @@
@impl true
def get_head(config, prefix) do
path = Path.join([config.root, prefix, "HEAD"])
path = safe_path(config.root, Path.join([prefix, "HEAD"]))
case File.read(path) do
{:ok, data} -> {:ok, String.trim(data)}
@@ -250,6 +250,6 @@
@impl true
def put_head(config, prefix, target) do
path = safe_path(config.root, Path.join([prefix, "HEAD"]))
path = Path.join([config.root, prefix, "HEAD"])
atomic_write(path, target <> "\n")
end
lib/ex_git_objectstore/storage/s3.ex +14 −5
@@ -121,7 +121,7 @@
@impl true
def get_ref(config, prefix, ref_path) do
key = "#{prefix}/#{ref_path}"
key = safe_key("#{prefix}/#{ref_path}")
case s3_get(config, key) do
{:ok, data} -> {:ok, String.trim(data)}
@@ -131,7 +131,7 @@
@impl true
def put_ref(config, prefix, ref_path, new_sha, old_sha) do
key = safe_key("#{prefix}/#{ref_path}")
key = "#{prefix}/#{ref_path}"
if old_sha do
# CAS: read current value first
@@ -156,6 +156,6 @@
@impl true
def delete_ref(config, prefix, ref_path) do
key = "#{prefix}/#{ref_path}"
key = safe_key("#{prefix}/#{ref_path}")
s3_delete(config, key)
end
@@ -208,11 +208,20 @@
defp object_key(prefix, sha) do
<<dir::binary-size(2), rest::binary>> = sha
safe_key("#{prefix}/objects/#{dir}/#{rest}")
"#{prefix}/objects/#{dir}/#{rest}"
end
defp pack_key(prefix, pack_sha, ext) do
safe_key("#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}")
end
"#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}"
# Prevent path traversal in S3 keys
defp safe_key(key) do
if String.contains?(key, "..") do
raise ArgumentError, "path traversal detected in S3 key"
end
key
end
defp s3_get(config, key) do
mix.exs +2 −1
@@ -94,7 +94,8 @@
{:hackney, "~> 1.18"},
{:jason, "~> 1.4"},
{:ex_doc, "~> 0.34", only: :dev, runtime: false},
{:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}
{:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false},
{:credo, "~> 1.7", only: [:dev, :test], runtime: false}
]
end
end
mix.lock +3 −0
@@ -1,11 +1,14 @@
%{
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"},
"credo": {:hex, :credo, "1.7.16", "a9f1389d13d19c631cb123c77a813dbf16449a2aebf602f590defa08953309d4", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0562af33756b21f248f066a9119e3890722031b6d199f22e3cf95550e4f1579"},
"dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"},
"earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"},
"erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"},
"ex_aws": {:hex, :ex_aws, "2.6.1", "194582c7b09455de8a5ab18a0182e6dd937d53df82be2e63c619d01bddaccdfa", [:mix], [{:configparser_ex, "~> 5.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67842a08c90a1d9a09dbe4ac05754175c7ca253abe4912987c759395d4bd9d26"},
"ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"},
"ex_doc": {:hex, :ex_doc, "0.40.1", "67542e4b6dde74811cfd580e2c0149b78010fd13001fda7cfeb2b2c2ffb1344d", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "bcef0e2d360d93ac19f01a85d58f91752d930c0a30e2681145feea6bd3516e00"},
"file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
"hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
test/ex_git_objectstore/object/blob_test.exs +18 −3
@@ -47,10 +47,25 @@
end
end
describe "spec compliance - header size validation" do
test "decode_raw rejects size mismatch" do
# Construct a raw object with wrong size in header
# Header says 10 bytes but content is only 5
raw = "blob 10\0hello"
result = Object.decode_raw(raw)
assert {:error, {:size_mismatch, expected: 10, actual: 5}} = result
end
test "decode_raw accepts correct size" do
raw = "blob 5\0hello"
assert {:ok, %Blob{content: "hello"}} = Object.decode_raw(raw)
end
end
describe "blob round-trip" do
test "encode then decode returns original" do
blob = Blob.from_content("test content here")
encoded = Object.encode(blob)
encoded = Object.encode!(blob)
assert {:ok, decoded} = Object.decode(encoded)
assert %Blob{content: "test content here"} = decoded
end
@@ -58,7 +73,7 @@
test "round-trip preserves binary content" do
content = <<0, 1, 2, 255, 128, 64>>
blob = Blob.from_content(content)
encoded = Object.encode!(blob)
encoded = Object.encode(blob)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.content == content
end
@@ -66,7 +81,7 @@
test "round-trip preserves large content" do
content = String.duplicate("x", 100_000)
blob = Blob.from_content(content)
encoded = Object.encode(blob)
encoded = Object.encode!(blob)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.content == content
end
test/ex_git_objectstore/object/commit_test.exs +60 −4
@@ -30,7 +30,7 @@
message: "Initial commit\n"
}
encoded = Object.encode(commit)
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert %Commit{} = decoded
assert decoded.tree == @tree_sha
@@ -51,7 +51,7 @@
message: "Second commit\n"
}
encoded = Object.encode(commit)
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.parents == [parent_sha]
end
@@ -70,7 +70,7 @@
message: "Merge commit\n"
}
encoded = Object.encode(commit)
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.parents == parents
end
@@ -112,6 +112,62 @@
assert content == expected
end
test "encode and decode commit with GPG signature (continuation lines)" do
gpgsig =
"-----BEGIN PGP SIGNATURE-----\n" <>
"iQEzBAABCAAdFiEEYp1234567890\n" <>
"abcdef1234567890==\n" <>
"-----END PGP SIGNATURE-----"
commit = %Commit{
tree: @tree_sha,
parents: [],
author: "Test User <test@example.com> 1234567890 +0000",
committer: "Test User <test@example.com> 1234567890 +0000",
gpgsig: gpgsig,
message: "Signed commit\n"
}
# Encode and verify continuation lines format
content = Commit.encode_content(commit)
lines = String.split(content, "\n")
# The gpgsig header should use continuation lines (space prefix)
gpgsig_idx = Enum.find_index(lines, &String.starts_with?(&1, "gpgsig "))
assert gpgsig_idx != nil
# Lines after the first gpgsig line should start with space (continuation)
continuation_lines =
lines
|> Enum.drop(gpgsig_idx + 1)
|> Enum.take_while(&String.starts_with?(&1, " "))
assert length(continuation_lines) >= 2
# Roundtrip: encode then decode preserves the signature
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.gpgsig != nil
# The decoded gpgsig should contain the continuation line prefix
assert String.contains?(decoded.gpgsig, "BEGIN PGP SIGNATURE")
assert String.contains?(decoded.gpgsig, "END PGP SIGNATURE")
end
test "encode and decode commit with extra headers" do
commit = %Commit{
tree: @tree_sha,
parents: [],
author: "Test User <test@example.com> 1234567890 +0000",
committer: "Test User <test@example.com> 1234567890 +0000",
extra_headers: [{"mergetag", "some-tag-data"}],
message: "Merge commit\n"
}
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert [{"mergetag", "some-tag-data"}] = decoded.extra_headers
end
test "encode and decode preserves multi-line message" do
commit = %Commit{
tree: @tree_sha,
@@ -121,7 +177,7 @@
message: "First line\n\nDetailed description\nwith multiple lines\n"
}
encoded = Object.encode(commit)
encoded = Object.encode!(commit)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.message == "First line\n\nDetailed description\nwith multiple lines\n"
end
test/ex_git_objectstore/object/tag_test.exs +34 −2
@@ -30,7 +30,7 @@
message: "Release v1.0.0\n"
}
encoded = Object.encode(tag)
encoded = Object.encode!(tag)
assert {:ok, decoded} = Object.decode(encoded)
assert %Tag{} = decoded
assert decoded.object == @commit_sha
@@ -77,6 +77,38 @@
assert content == expected
end
test "encode and decode tag without tagger (optional tagger)" do
tag = %Tag{
object: @commit_sha,
type: "commit",
tag: "v0.1.0",
tagger: nil,
message: "Lightweight-style annotated tag\n"
}
encoded = Object.encode!(tag)
assert {:ok, decoded} = Object.decode(encoded)
assert %Tag{} = decoded
assert decoded.tagger == nil
assert decoded.tag == "v0.1.0"
assert decoded.message == "Lightweight-style annotated tag\n"
end
test "encode and decode tag with extra headers" do
tag = %Tag{
object: @commit_sha,
type: "commit",
tag: "v1.1.0",
tagger: "Test User <test@example.com> 1234567890 +0000",
extra_headers: [{"some-header", "some-value"}],
message: "Tag with extra headers\n"
}
encoded = Object.encode!(tag)
assert {:ok, decoded} = Object.decode(encoded)
assert [{"some-header", "some-value"}] = decoded.extra_headers
end
test "encode and decode tag with multi-line message" do
tag = %Tag{
object: @commit_sha,
@@ -86,7 +118,7 @@
message: "Major release\n\nBreaking changes:\n- Changed API\n- New format\n"
}
encoded = Object.encode(tag)
encoded = Object.encode!(tag)
assert {:ok, decoded} = Object.decode(encoded)
assert decoded.message ==
test/ex_git_objectstore/object/tree_test.exs +128 −5
@@ -25,7 +25,7 @@
%{mode: "100644", name: "hello.txt", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"}
])
encoded = Object.encode(tree)
encoded = Object.encode!(tree)
assert {:ok, decoded} = Object.decode(encoded)
assert %Tree{entries: [entry]} = decoded
assert entry.mode == "100644"
@@ -41,7 +41,7 @@
%{mode: "40000", name: "subdir", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"}
])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode()
assert {:ok, decoded} = tree |> Object.encode() |> Object.decode()
# Should be sorted: a.txt, b.txt, subdir
assert [first, second, third] = decoded.entries
assert first.name == "a.txt"
@@ -55,7 +55,7 @@
%{mode: "40000", name: "lib", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"}
])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode()
assert {:ok, decoded} = tree |> Object.encode() |> Object.decode()
assert [entry] = decoded.entries
assert entry.mode == "40000"
end
@@ -66,7 +66,7 @@
%{mode: "100755", name: "run.sh", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"}
])
assert {:ok, decoded} = tree |> Object.encode() |> Object.decode()
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode()
assert [entry] = decoded.entries
assert entry.mode == "100755"
end
@@ -77,7 +77,7 @@
%{mode: "120000", name: "link", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"}
])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode()
assert {:ok, decoded} = tree |> Object.encode() |> Object.decode()
assert [entry] = decoded.entries
assert entry.mode == "120000"
end
@@ -113,5 +113,128 @@
names = Enum.map(tree.entries, & &1.name)
# "aa/" < "ab" < "ac"
assert names == ["aa", "ab", "ac"]
end
end
describe "spec compliance - mode canonicalization" do
test "canonicalizes short modes" do
tree =
Tree.new([
%{mode: "644", name: "a.txt", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
assert [%{mode: "100644"}] = tree.entries
end
test "canonicalizes padded modes" do
tree =
Tree.new([
%{mode: "0100644", name: "a.txt", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
assert [%{mode: "100644"}] = tree.entries
end
test "canonicalizes executable mode" do
tree =
Tree.new([
%{mode: "755", name: "run.sh", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
assert [%{mode: "100755"}] = tree.entries
end
test "canonicalizes directory mode" do
tree =
Tree.new([
%{mode: "040000", name: "dir", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"}
])
assert [%{mode: "40000"}] = tree.entries
end
test "canonicalizes gitlink (submodule) mode" do
tree =
Tree.new([
%{mode: "0160000", name: "submod", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
assert [%{mode: "160000"}] = tree.entries
end
end
describe "spec compliance - entry name validation" do
test "rejects empty name" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: "", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "rejects dot entry" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: ".", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "rejects dotdot entry" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: "..", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "rejects .git entry (case-insensitive)" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: ".GIT", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "rejects slash in name" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: "a/b", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "rejects null byte in name" do
assert_raise ArgumentError, ~r/invalid tree entry name/, fn ->
Tree.new([
%{mode: "100644", name: "a\0b", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
end
end
test "allows valid names" do
tree =
Tree.new([
%{
mode: "100644",
name: "valid-file.txt",
sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
},
%{mode: "100644", name: ".gitignore", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"},
%{mode: "100644", name: ".gitmodules", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}
])
assert length(tree.entries) == 3
end
end
describe "spec compliance - directory_mode?" do
test "identifies directory modes" do
assert Tree.directory_mode?("40000")
assert Tree.directory_mode?("040000")
assert Tree.directory_mode?("0040000")
refute Tree.directory_mode?("100644")
refute Tree.directory_mode?("100755")
refute Tree.directory_mode?("160000")
end
end
test/ex_git_objectstore/pack/index_test.exs +56 −0
@@ -47,6 +47,62 @@
end
end
describe "spec compliance - checksum and sorting" do
test "generated index has valid checksum" do
# Use Writer to generate a pack+index, then verify index parses
blob = ExGitObjectstore.Object.Blob.from_content("test\n")
sha = ExGitObjectstore.Object.hash(blob)
{_pack_data, idx_data, _pack_sha} =
ExGitObjectstore.Pack.Writer.generate_with_index([{:blob, "test\n", sha}])
# Verify the index checksum by checking the last 20 bytes
body_len = byte_size(idx_data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = idx_data
assert checksum == :crypto.hash(:sha, body)
# Parse should succeed (validates checksum internally)
assert {:ok, _index} = Index.parse(idx_data)
end
test "rejects index with corrupted checksum" do
blob = ExGitObjectstore.Object.Blob.from_content("test\n")
sha = ExGitObjectstore.Object.hash(blob)
{_pack_data, idx_data, _pack_sha} =
ExGitObjectstore.Pack.Writer.generate_with_index([{:blob, "test\n", sha}])
# Corrupt the trailing checksum
body_len = byte_size(idx_data) - 20
<<body::binary-size(body_len), _checksum::binary-size(20)>> = idx_data
bad_checksum = :binary.copy(<<0>>, 20)
corrupted = <<body::binary, bad_checksum::binary>>
assert {:error, :index_checksum_mismatch} = Index.parse(corrupted)
end
test "SHAs in generated index are sorted" do
blob1 = ExGitObjectstore.Object.Blob.from_content("one\n")
sha1 = ExGitObjectstore.Object.hash(blob1)
blob2 = ExGitObjectstore.Object.Blob.from_content("two\n")
sha2 = ExGitObjectstore.Object.hash(blob2)
blob3 = ExGitObjectstore.Object.Blob.from_content("three\n")
sha3 = ExGitObjectstore.Object.hash(blob3)
objects = [
{:blob, "one\n", sha1},
{:blob, "two\n", sha2},
{:blob, "three\n", sha3}
]
{_pack_data, idx_data, _pack_sha} =
ExGitObjectstore.Pack.Writer.generate_with_index(objects)
assert {:ok, index} = Index.parse(idx_data)
assert index.shas == Enum.sort(index.shas)
end
end
# -- Helpers --
defp create_packed_repo do
test/ex_git_objectstore/pack/reader_test.exs +74 −0
@@ -67,6 +67,80 @@
end
end
describe "spec compliance - pack checksum verification" do
test "verify_pack_checksum succeeds on valid pack" do
content = "hello world\n"
blob = ExGitObjectstore.Object.Blob.from_content(content)
sha = ExGitObjectstore.Object.hash(blob)
{pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
assert :ok = Reader.verify_pack_checksum(pack_data)
end
test "verify_pack_checksum detects corruption" do
content = "hello world\n"
blob = ExGitObjectstore.Object.Blob.from_content(content)
sha = ExGitObjectstore.Object.hash(blob)
{pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Corrupt one byte in the middle
mid = div(byte_size(pack_data), 2)
<<before::binary-size(mid), byte, after_bytes::binary>> = pack_data
corrupted = <<before::binary, Bitwise.bxor(byte, 0xFF), after_bytes::binary>>
assert {:error, :pack_checksum_mismatch} = Reader.verify_pack_checksum(corrupted)
end
test "verify_pack_checksum rejects too-short data" do
assert {:error, :pack_too_short} = Reader.verify_pack_checksum(<<1, 2, 3>>)
end
test "parse rejects pack with corrupted checksum" do
content = "hello world\n"
blob = ExGitObjectstore.Object.Blob.from_content(content)
sha = ExGitObjectstore.Object.hash(blob)
{pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Corrupt the trailing checksum
body_len = byte_size(pack_data) - 20
<<body::binary-size(body_len), _checksum::binary-size(20)>> = pack_data
bad_checksum = :binary.copy(<<0>>, 20)
corrupted = <<body::binary, bad_checksum::binary>>
assert {:error, :pack_checksum_mismatch} = Reader.parse(corrupted)
end
end
describe "spec compliance - version support" do
test "accepts version 2 packs" do
content = "test\n"
blob = ExGitObjectstore.Object.Blob.from_content(content)
sha = ExGitObjectstore.Object.hash(blob)
{pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Verify it's version 2
<<"PACK", 2::unsigned-big-32, _rest::binary>> = pack_data
assert {:ok, _entries} = Reader.parse(pack_data)
end
end
describe "spec compliance - SHA index building" do
test "build_sha_index returns SHA-to-offset map" do
content = "hello\n"
blob = ExGitObjectstore.Object.Blob.from_content(content)
sha = ExGitObjectstore.Object.hash(blob)
{pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
assert {:ok, index} = Reader.build_sha_index(pack_data)
assert Map.has_key?(index, sha)
assert is_integer(Map.get(index, sha))
end
test "build_sha_index rejects invalid header" do
assert {:error, :invalid_pack_header} = Reader.build_sha_index("not a pack")
end
end
# -- Helpers --
defp create_packed_repo_with_objects do
test/ex_git_objectstore/protocol/pkt_line_test.exs +57 −0
@@ -141,6 +141,63 @@
end
end
describe "spec compliance - max length" do
test "encode raises on oversized data" do
# Max total pkt-line is 65520 bytes (0xFFF0)
# encode adds 4-byte prefix + 1-byte newline, so max data is 65515 chars
big_data = String.duplicate("x", 65516)
assert_raise ArgumentError, ~r/pkt-line too long/, fn ->
PktLine.encode(big_data)
end
end
test "encode_raw raises on oversized data" do
# encode_raw adds 4-byte prefix, so max payload is 65516 bytes
big_data = :binary.copy(<<0>>, 65517)
assert_raise ArgumentError, ~r/pkt-line too long/, fn ->
PktLine.encode_raw(big_data)
end
end
test "encode allows data at exact max length" do
# Max data + newline + 4 prefix = 65520
# If data already has newline: 65516 bytes data = 65520 total
data = String.duplicate("x", 65515) <> "\n"
result = PktLine.encode(data)
assert byte_size(result) == 65520
end
test "encode_raw allows data at exact max payload" do
# 65516 bytes + 4 prefix = 65520
data = :binary.copy(<<0>>, 65516)
result = PktLine.encode_raw(data)
assert byte_size(result) == 65520
end
end
describe "spec compliance - trailing LF stripping" do
test "strips exactly one trailing LF" do
# Data "hello\n\n" (2 newlines at end)
# Encoding: 4 prefix + 7 data = 11 = 000b
encoded = "000bhello\n\n"
assert {:ok, {:data, "hello\n"}, ""} = PktLine.decode_one(encoded)
end
test "does not strip non-LF trailing bytes" do
encoded = PktLine.encode_raw("hello")
assert {:ok, {:data, "hello"}, ""} = PktLine.decode_one(encoded)
end
test "handles empty payload" do
# Length 4 means 0 data bytes, but min is actually 5 for data
# encode_raw of empty string: "0004" (4 bytes prefix, 0 data = 4 total)
encoded = PktLine.encode_raw("")
assert {:ok, {:data, ""}, ""} = PktLine.decode_one(encoded)
end
end
describe "sideband" do
test "decode sideband band 1 (pack data)" do
assert {:pack, "data"} = PktLine.decode_sideband(<<1, "data">>)
test/ex_git_objectstore/protocol/receive_pack_test.exs +111 −2
@@ -70,9 +70,118 @@
_ -> false
end)
assert length(data_lines) >= 1
# HEAD first (spec: HEAD MUST appear first), then refs/heads/main
assert length(data_lines) >= 2
{:data, first_line} = hd(data_lines)
assert String.contains?(first_line, commit_sha)
assert String.contains?(first_line, "HEAD")
# refs/heads/main should appear in subsequent lines
all_text = Enum.map(data_lines, fn {:data, d} -> d end) |> Enum.join("\n")
assert String.contains?(all_text, "refs/heads/main")
end
end
describe "spec compliance - advertisement format" do
test "HEAD appears first in advertisement with refs" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
blob = Blob.from_content("hello\n")
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: "init\n"
}
{:ok, commit_sha} = Object.write(repo, commit)
:ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = ReceivePack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
data_lines = for {:data, d} <- packets, do: d
# First line must be HEAD
first = hd(data_lines)
assert String.contains?(first, "HEAD")
assert String.contains?(first, commit_sha)
end
test "capabilities immediately follow NUL (no space)" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, _state} = ReceivePack.init(repo)
{:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
[_ref_part, caps_part] = String.split(first_line, "\0", parts: 2)
refute String.starts_with?(caps_part, " ")
assert String.contains?(caps_part, "report-status")
assert String.contains?(caps_part, "delete-refs")
end
test "empty repo advertises zero SHA with capabilities" do
repo = RepoHelper.memory_repo()
assert String.contains?(first_line, "refs/heads/main")
ExGitObjectstore.init(repo)
{advert, _state} = ReceivePack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
data_lines = for {:data, d} <- packets, do: d
first = hd(data_lines)
zero_sha = String.duplicate("0", 40)
assert String.starts_with?(first, zero_sha)
assert String.contains?(first, "capabilities^{}")
end
end
describe "spec compliance - client capabilities" do
test "parses client capabilities from first command line" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
blob = Blob.from_content("hello\n")
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: "init\n"
}
{:ok, commit_sha} = Object.write(repo, commit)
{_advert, state} = ReceivePack.init(repo)
# Send command with capabilities after NUL
zero = String.duplicate("0", 40)
cmd_with_caps = "#{zero} #{commit_sha} refs/heads/main\0report-status delete-refs"
commands = PktLine.encode_raw(cmd_with_caps <> "\n") <> PktLine.flush()
objects = [
{:blob, "hello\n", blob_sha},
{:tree, Tree.encode_content(tree), tree_sha},
{:commit, Commit.encode_content(commit), commit_sha}
]
{pack_data, _} = Writer.generate(objects)
{response, state} = ReceivePack.feed(state, commands <> pack_data)
assert ReceivePack.done?(state)
# Client sent report-status capability, so we should get a report
{:ok, packets, _} = PktLine.decode(response)
data_lines = for {:data, d} <- packets, do: d
assert "unpack ok" in data_lines
end
end
test/ex_git_objectstore/protocol/upload_pack_test.exs +61 −0
@@ -68,6 +68,67 @@
end
end
describe "spec compliance - advertisement format" do
test "HEAD appears first in advertisement" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{commit_sha, _, _} = create_commit(repo, "hello\n", "init\n")
:ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = UploadPack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
data_lines = for {:data, d} <- packets, do: d
# First line must be HEAD (spec requirement)
first = hd(data_lines)
assert String.contains?(first, "HEAD")
assert String.contains?(first, commit_sha)
end
test "capabilities immediately follow NUL (no space)" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo)
# Parse raw bytes: the first pkt-line should contain \0 followed immediately by caps
{:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
# There should be a NUL byte separating ref info from capabilities
assert String.contains?(first_line, "\0")
# After NUL, the next character should NOT be a space
[_ref_part, caps_part] = String.split(first_line, "\0", parts: 2)
refute String.starts_with?(caps_part, " ")
end
test "includes symref capability" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo)
{:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
[_ref_part, caps_part] = String.split(first_line, "\0", parts: 2)
assert String.contains?(caps_part, "symref=HEAD:refs/heads/main")
end
test "empty repo advertises zero SHA with capabilities" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
data_lines = for {:data, d} <- packets, do: d
first = hd(data_lines)
zero_sha = String.duplicate("0", 40)
assert String.starts_with?(first, zero_sha)
assert String.contains?(first, "capabilities^{}")
end
end
describe "feed - clone (wants only, no haves)" do
test "sends packfile with all objects for a clone" do
repo = RepoHelper.memory_repo()
test/ex_git_objectstore/security_test.exs +255 −0
@@ -1,0 +1,255 @@
# Copyright 2026 Cole Christensen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
defmodule ExGitObjectstore.SecurityTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.{Object, Ref}
alias ExGitObjectstore.Object.{Blob, Commit, Tag, Tree}
alias ExGitObjectstore.Pack.Delta
alias ExGitObjectstore.Storage.Filesystem
alias ExGitObjectstore.Repo
alias ExGitObjectstore.Test.RepoHelper
describe "path traversal prevention in filesystem ref operations" do
setup do
tmp_dir = System.tmp_dir!()
root = Path.join(tmp_dir, "sec_test_#{:erlang.unique_integer([:positive])}")
File.mkdir_p!(root)
repo = Repo.new("test-repo", storage: {Filesystem, %{root: root}})
on_exit(fn -> File.rm_rf!(root) end)
%{repo: repo, root: root}
end
test "put_ref rejects path traversal via ref name", %{repo: repo} do
sha = String.duplicate("a", 40)
assert {:error, {:invalid_ref_name, _, :dotdot}} =
Ref.put(repo, "refs/heads/../../etc/passwd", sha, nil)
end
test "get rejects path traversal via ref name", %{repo: repo} do
assert {:error, {:invalid_ref_name, _, :dotdot}} =
Ref.get(repo, "refs/heads/../../../etc/shadow")
end
test "delete rejects path traversal via ref name", %{repo: repo} do
assert {:error, {:invalid_ref_name, _, :dotdot}} =
Ref.delete(repo, "refs/heads/../../etc/important")
end
test "list rejects path traversal via prefix", %{repo: repo} do
assert {:error, {:invalid_ref_prefix, _, :dotdot}} =
Ref.list(repo, "refs/../../")
end
test "filesystem safe_path blocks traversal in ref paths directly", %{root: root} do
config = %{root: root}
# Need enough ".." to escape root entirely (root/repos/test/../../../../esc)
assert_raise ArgumentError, ~r/path traversal/, fn ->
Filesystem.put_ref(
config,
"repos/test",
"../../../../etc/passwd",
String.duplicate("a", 40),
nil
)
end
end
end
describe "ref name validation on all public functions" do
test "get validates ref name" do
repo = RepoHelper.memory_repo()
assert {:error, {:invalid_ref_name, "", :empty}} = Ref.get(repo, "")
end
test "delete validates ref name" do
repo = RepoHelper.memory_repo()
assert {:error, {:invalid_ref_name, "", :empty}} = Ref.delete(repo, "")
end
test "list validates ref prefix" do
repo = RepoHelper.memory_repo()
assert {:error, {:invalid_ref_prefix, _, :null_byte}} = Ref.list(repo, "refs/\0bad/")
end
end
describe "decompression size limit" do
test "decode rejects oversized decompressed objects" do
# Create a large blob and compress it
large_content = :binary.copy(<<0>>, 200 * 1024 * 1024)
blob = Blob.from_content(large_content)
# This should fail because the decompressed object exceeds the 128MB limit
case Object.encode(blob) do
{:ok, encoded} ->
assert {:error, {:object_too_large, _, _}} = Object.decode(encoded)
{:error, _} ->
# Compression itself may fail with very large data, that's ok
:ok
end
end
end
describe "encode!/1 convention" do
test "encode! raises on failure" do
# Normal encoding should work fine
blob = Blob.from_content("hello")
assert is_binary(Object.encode!(blob))
end
test "encode/1 returns ok/error tuple" do
blob = Blob.from_content("hello")
assert {:ok, encoded} = Object.encode(blob)
assert is_binary(encoded)
end
end
describe "@enforce_keys on structs" do
test "Commit requires tree, author, committer, message via struct!" do
assert_raise ArgumentError, ~r/the following keys must also be given/, fn ->
struct!(Commit, %{})
end
end
test "Tag requires object, type, tag, tagger, message via struct!" do
assert_raise ArgumentError, ~r/the following keys must also be given/, fn ->
struct!(Tag, %{})
end
end
test "Commit with all required keys succeeds" do
commit =
struct!(Commit, %{
tree: String.duplicate("a", 40),
author: "Test <t@t.com> 0 +0000",
committer: "Test <t@t.com> 0 +0000",
message: "test\n"
})
assert commit.tree == String.duplicate("a", 40)
end
test "Tag with all required keys succeeds" do
tag =
struct!(Tag, %{
object: String.duplicate("a", 40),
type: "commit",
tag: "v1.0",
tagger: "Test <t@t.com> 0 +0000",
message: "test\n"
})
assert tag.tag == "v1.0"
end
end
describe "tree mode canonicalization" do
test "644 is normalized to 100644" do
tree = Tree.new([%{mode: "644", name: "f.txt", sha: String.duplicate("a", 40)}])
assert hd(tree.entries).mode == "100644"
end
test "755 is normalized to 100755" do
tree = Tree.new([%{mode: "755", name: "f.sh", sha: String.duplicate("a", 40)}])
assert hd(tree.entries).mode == "100755"
end
test "040000 is normalized to 40000" do
tree = Tree.new([%{mode: "040000", name: "dir", sha: String.duplicate("a", 40)}])
assert hd(tree.entries).mode == "40000"
end
test "standard modes are unchanged" do
sha = String.duplicate("a", 40)
tree =
Tree.new([
%{mode: "100644", name: "a.txt", sha: sha},
%{mode: "100755", name: "b.sh", sha: sha},
%{mode: "40000", name: "dir", sha: sha},
%{mode: "120000", name: "link", sha: sha}
])
modes = Enum.map(tree.entries, & &1.mode)
assert "100644" in modes
assert "100755" in modes
assert "40000" in modes
assert "120000" in modes
end
end
describe "delta safety" do
test "truncated delta copy command returns error" do
base = "hello world"
# Build a delta that tries to read copy bytes from an empty buffer
# varint for base_size=11, target_size=5, then copy cmd with all offset bits set but no data
base_size_varint = <<11>>
target_size_varint = <<5>>
# Copy command: MSB set, bits 0-3 set (4 offset bytes expected), but no data follows
copy_cmd = <<0xFF>>
delta = base_size_varint <> target_size_varint <> copy_cmd
assert {:error, :truncated_delta_copy} = Delta.apply(base, delta)
end
end
describe "receive-pack buffer limit" do
test "rejects oversized pack data" do
alias ExGitObjectstore.Protocol.ReceivePack
alias ExGitObjectstore.Protocol.PktLine
repo = RepoHelper.memory_repo()
{_advert, state} = ReceivePack.init(repo)
sha = String.duplicate("a", 40)
zero = String.duplicate("0", 40)
commands = PktLine.encode("#{zero} #{sha} refs/heads/main") <> PktLine.flush()
# Feed commands first
{_resp, state} = ReceivePack.feed(state, commands)
# Now feed data that exceeds the limit (256 MB)
# We can't actually allocate 256MB in a test, but we can test the guard
# by feeding chunks that accumulate
chunk = :binary.copy("x", 1024 * 1024)
# Feed until we exceed the limit or get an error
result =
Enum.reduce_while(1..300, state, fn _i, st ->
{resp, new_state} = ReceivePack.feed(st, chunk)
if ReceivePack.done?(new_state) do
{:halt, {:done, resp}}
else
{:cont, new_state}
end
end)
case result do
{:done, resp} ->
# Should have error about pack too large
assert resp =~ "pack_too_large"
_state ->
flunk("Should have rejected oversized pack")
end
end
end
end