|
|
lib/ex_git_objectstore/diff.ex
|
+2
−7
|
@@ -86,13 +86,8 @@ {:ok, changes} <- diff_trees(repo, old_commit.tree, new_commit.tree) do file_diffs = Enum.map(changes, fn change -> case diff_blobs(repo, change.old_sha, change.new_sha, opts) do {:ok, hunks} -> %{path: change.path, status: change.status, hunks: hunks}
{:error, _} -> %{path: change.path, status: change.status, hunks: []} end {:ok, hunks} = diff_blobs(repo, change.old_sha, change.new_sha, opts) %{path: change.path, status: change.status, hunks: hunks} end)
{:ok, file_diffs}
|
|
|
lib/ex_git_objectstore/object.ex
|
+51
−16
|
@@ -23,5 +23,8 @@ alias ExGitObjectstore.Object.{Blob, Commit, Tree, Tag} alias ExGitObjectstore.Repo
# Maximum decompressed object size (128 MB) @max_object_size 128 * 1024 * 1024
@type object_type :: :blob | :commit | :tree | :tag @type t :: Blob.t() | Commit.t() | Tree.t() | Tag.t() @@ -47,22 +50,23 @@
@doc """ Encode a git object to its stored format (zlib compressed). Raises on compression failure (extremely rare). Returns `{:ok, compressed}` or `{:error, reason}`. """ @spec encode(t()) :: binary() @spec encode(t()) :: {:ok, binary()} | {:error, term()} def encode(object) do case zlib_compress(encode_raw(object)) do zlib_compress(encode_raw(object)) {:ok, compressed} -> compressed {:error, reason} -> raise "zlib compression failed: #{inspect(reason)}" end end
@doc """ Safe version of encode/1 — returns `{:ok, compressed}` or `{:error, reason}`. Encode a git object to its stored format (zlib compressed). Raises on compression failure. """ @spec safe_encode(t()) :: {:ok, binary()} | {:error, term()} def safe_encode(object) do @spec encode!(t()) :: binary() def encode!(object) do case encode(object) do {:ok, compressed} -> compressed {:error, reason} -> raise "zlib compression failed: #{inspect(reason)}" zlib_compress(encode_raw(object)) end end
@doc """ @@ -82,8 +86,13 @@ @spec decode_raw(binary()) :: {:ok, t()} | {:error, term()} def decode_raw(raw) do case parse_header(raw) do {:ok, type, size, content} -> # Validate that header size matches actual content length {:ok, type, _size, content} -> decode_typed(type, content) if byte_size(content) != size do {:error, {:size_mismatch, expected: size, actual: byte_size(content)}} else decode_typed(type, content) end
{:error, _} = err -> err @@ -96,7 +105,23 @@ @spec read(Repo.t(), String.t()) :: {:ok, t()} | {:error, term()} def read(%Repo{} = repo, sha) do case Repo.storage_call(repo, :get_object, [sha]) do {:ok, data} -> # Verify SHA-1 integrity: decompress, hash, and compare to requested SHA case safe_decompress(data) do {:ok, raw} -> actual_sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
if actual_sha != sha do {:error, {:sha_mismatch, expected: sha, actual: actual_sha}} else decode_raw(raw) end
{:error, _} = err -> err end
{:error, _} = err -> err {:ok, data} -> decode(data) {:error, _} = err -> err end end @@ -108,7 +133,7 @@ def write(%Repo{} = repo, object) do sha = hash(object)
case safe_encode(object) do case encode(object) do {:ok, compressed} -> case Repo.storage_call(repo, :put_object, [sha, compressed]) do :ok -> {:ok, sha} @@ -186,13 +211,23 @@ end
defp safe_decompress(data) do safe_decompress(data, @max_object_size) end
defp safe_decompress(data, max_size) do z = :zlib.open()
try do :zlib.inflateInit(z) result = :zlib.inflate(z, data) :zlib.inflateEnd(z) {:ok, IO.iodata_to_binary(result)} decompressed = IO.iodata_to_binary(result)
if byte_size(decompressed) > max_size do {:error, {:object_too_large, byte_size(decompressed), max_size}} else {:ok, decompressed} end rescue e -> {:error, {:decompress_failed, Exception.message(e)}} after
|
|
|
lib/ex_git_objectstore/object/commit.ex
|
+29
−11
|
@@ -35,6 +35,7 @@ extra_headers: [{String.t(), String.t()}] }
@enforce_keys [:tree, :author, :committer, :message] defstruct [ :tree, :author, @@ -59,7 +60,7 @@
lines = if commit.gpgsig do lines ++ [encode_multiline_header("gpgsig", commit.gpgsig)] lines ++ ["gpgsig #{commit.gpgsig}\n"] else lines end @@ -67,7 +68,7 @@ lines = lines ++ Enum.map(commit.extra_headers, fn {key, value} -> "#{key} #{value}\n" encode_multiline_header(key, value) end)
lines = lines ++ ["\n", commit.message] @@ -101,6 +102,23 @@ end) end
# Encode a header value that may contain newlines as continuation lines. # Per the git spec, continuation lines are prefixed with a space character. defp encode_multiline_header(key, value) do if String.contains?(value, "\n") do # Split on newlines and prefix continuation lines with space [first | rest] = String.split(value, "\n")
lines = ["#{key} #{first}\n"] ++ Enum.map(rest, fn line -> " #{line}\n" end)
IO.iodata_to_binary(lines) else "#{key} #{value}\n" end end
# Handle GPG signature continuation lines (start with space) defp fold_continuation_lines(lines) do lines @@ -118,27 +136,27 @@ end
defp build_commit(headers, message) do fields = commit = Enum.reduce(headers, %__MODULE__{message: message}, fn Enum.reduce(headers, %{message: message, parents: [], extra_headers: []}, fn {"tree", sha}, acc -> Map.put(acc, :tree, sha) %{acc | tree: sha}
{"parent", sha}, acc -> %{acc | parents: acc.parents ++ [sha]} Map.update(acc, :parents, [sha], &(&1 ++ [sha]))
{"author", value}, acc -> %{acc | author: value} Map.put(acc, :author, value)
{"committer", value}, acc -> Map.put(acc, :committer, value) %{acc | committer: value}
{"gpgsig", value}, acc -> %{acc | gpgsig: value} Map.put(acc, :gpgsig, value)
{key, value}, acc -> Map.update(acc, :extra_headers, [{key, value}], &(&1 ++ [{key, value}])) %{acc | extra_headers: acc.extra_headers ++ [{key, value}]} end)
{:ok, commit} {:ok, struct!(__MODULE__, fields)} end end
|
|
|
lib/ex_git_objectstore/object/tag.ex
|
+60
−17
|
@@ -29,25 +29,42 @@ object: String.t(), type: String.t(), tag: String.t(), tagger: String.t(), message: String.t() tagger: String.t() | nil, message: String.t(), extra_headers: [{String.t(), String.t()}] }
# tagger is optional — older git versions created tags without it @enforce_keys [:object, :type, :tag, :message] defstruct [:object, :type, :tag, :tagger, :message, extra_headers: []] defstruct [:object, :type, :tag, :tagger, :message]
@doc """ Encode tag to its content format. """ @spec encode_content(t()) :: binary() def encode_content(%__MODULE__{} = tag) do IO.iodata_to_binary([ lines = [ "object #{tag.object}\n", "type #{tag.type}\n", "tag #{tag.tag}\n" ]
lines = if tag.tagger do lines ++ ["tagger #{tag.tagger}\n"] else lines end
lines = lines ++ Enum.map(tag.extra_headers || [], fn {key, value} -> "#{key} #{value}\n" end)
lines = lines ++ ["\n", tag.message]
IO.iodata_to_binary(lines) "tag #{tag.tag}\n", "tagger #{tag.tagger}\n", "\n", tag.message ]) end
@doc """ @@ -60,6 +77,7 @@ headers = headers_str |> String.split("\n") |> fold_continuation_lines() |> Enum.map(fn line -> case String.split(line, " ", parts: 2) do [key, value] -> {key, value} @@ -67,19 +85,44 @@ end end)
fields = Enum.reduce(headers, %{message: message, extra_headers: []}, fn {"object", sha}, acc -> tag = Enum.reduce(headers, %__MODULE__{message: message}, fn {"object", sha}, acc -> %{acc | object: sha} {"type", type}, acc -> %{acc | type: type} {"tag", name}, acc -> %{acc | tag: name} {"tagger", value}, acc -> %{acc | tagger: value} _, acc -> acc Map.put(acc, :object, sha)
{"type", type}, acc -> Map.put(acc, :type, type)
{"tag", name}, acc -> Map.put(acc, :tag, name)
{"tagger", value}, acc -> Map.put(acc, :tagger, value)
{key, value}, acc -> Map.update(acc, :extra_headers, [{key, value}], &(&1 ++ [{key, value}])) end)
{:ok, struct!(__MODULE__, fields)} {:ok, tag}
_ -> {:error, :invalid_tag_format} end end
# Handle continuation lines (start with space) for multiline headers defp fold_continuation_lines(lines) do lines |> Enum.reduce([], fn line, acc -> if String.starts_with?(line, " ") do case acc do [prev | rest] -> [prev <> "\n" <> line | rest] [] -> [line] end else [line | acc] end end) |> Enum.reverse() end end
|
|
|
lib/ex_git_objectstore/object/tree.ex
|
+46
−3
|
@@ -28,13 +28,53 @@ @doc """ Create a tree from a list of entries. Entries are sorted according to git's tree sorting rules. Modes are canonicalized to their standard git representations. """ @spec new([entry()]) :: t() def new(entries) do entries = Enum.map(entries, &canonicalize_entry/1)
# Validate entry names Enum.each(entries, fn entry -> case validate_entry_name(entry.name) do :ok -> :ok {:error, reason} -> raise ArgumentError, "invalid tree entry name: #{inspect(reason)}" end end)
%__MODULE__{entries: sort_entries(entries)} end
defp canonicalize_entry(%{mode: mode} = entry) do %{entry | mode: canonicalize_mode(mode)} end
@doc """ Canonicalize a git tree entry mode to its standard representation. """ @spec canonicalize_mode(String.t()) :: String.t() def canonicalize_mode("644"), do: "100644" def canonicalize_mode("755"), do: "100755" def canonicalize_mode("0644"), do: "100644" def canonicalize_mode("0755"), do: "100755" def canonicalize_mode("0100644"), do: "100644" def canonicalize_mode("0100755"), do: "100755" def canonicalize_mode("040000"), do: "40000" def canonicalize_mode("0040000"), do: "40000" # Gitlink (submodule) mode def canonicalize_mode("0160000"), do: "160000" def canonicalize_mode(mode), do: mode
@doc """ Returns true if the given mode represents a tree (directory). """ @spec directory_mode?(String.t()) :: boolean() def directory_mode?("40000"), do: true def directory_mode?("040000"), do: true def directory_mode?("0040000"), do: true def directory_mode?(_), do: false
@doc """ Validate a tree entry name according to git rules. Returns `:ok` or `{:error, reason}`. """ @@ -44,6 +84,7 @@ name == "" -> {:error, {:invalid_entry_name, name, :empty}} name == "." -> {:error, {:invalid_entry_name, name, :dot}} name == ".." -> {:error, {:invalid_entry_name, name, :dotdot}} String.downcase(name) == ".git" -> {:error, {:invalid_entry_name, name, :dotgit}} String.contains?(name, "/") -> {:error, {:invalid_entry_name, name, :slash}} String.contains?(name, <<0>>) -> {:error, {:invalid_entry_name, name, :null_byte}} true -> :ok @@ -98,11 +139,13 @@ end
# Git sorts tree entries by name, treating directories as if they have a trailing "/". # This matches git's base_name_compare() in tree-diff.c. defp sort_entries(entries) do Enum.sort_by(entries, fn entry -> case entry.mode do "40000" -> entry.name <> "/" _ -> entry.name if directory_mode?(entry.mode) do entry.name <> "/" else entry.name end end) end
|
|
|
lib/ex_git_objectstore/pack/delta.ex
|
+12
−3
|
@@ -49,6 +49,8 @@ {:error, {:size_mismatch, expected: target_size, got: byte_size(result_bin)}} end end catch {:error, _} = err -> err end
# Read a variable-length integer (used for base/target sizes in delta header) @@ -131,12 +133,19 @@ end
# Read one byte from data if the corresponding bit is set in cmd defp read_if_bit(cmd, bit, data) do defp read_if_bit(cmd, bit, <<byte, rest::binary>> = _data) do if Bitwise.band(cmd, Bitwise.bsl(1, bit)) != 0 do <<byte, rest::binary>> = data {byte, rest} else {0, <<byte, rest::binary>>} end end {0, data}
defp read_if_bit(cmd, bit, <<>>) do if Bitwise.band(cmd, Bitwise.bsl(1, bit)) != 0 do throw({:error, :truncated_delta_copy}) else {0, <<>>} end end end
|
|
|
lib/ex_git_objectstore/pack/index.ex
|
+51
−17
|
@@ -46,27 +46,61 @@ @max_pack_objects 10_000_000
@spec parse(binary()) :: {:ok, t()} | {:error, term()} def parse(<<@idx_magic, @idx_version, rest::binary>>) do with {:ok, fanout, rest} <- parse_fanout(rest), count = elem(fanout, 255), :ok <- validate_fanout(fanout), :ok <- validate_count(count), {:ok, shas, rest} <- parse_shas(rest, count), {:ok, _crcs, rest} <- parse_crcs(rest, count), {:ok, offsets_4, rest} <- parse_offsets_4(rest, count), {:ok, large_offsets} <- parse_large_offsets(rest, offsets_4), {:ok, offset_map} <- build_offset_map(shas, offsets_4, large_offsets) do {:ok, %__MODULE__{ fanout: fanout, shas: shas, offsets: offset_map, count: count }} def parse(<<@idx_magic, @idx_version, _rest::binary>> = idx_data) do # Verify index checksum first (PK4) with :ok <- verify_index_checksum(idx_data) do <<_magic_version::binary-size(8), rest::binary>> = idx_data
with {:ok, fanout, rest} <- parse_fanout(rest), count = elem(fanout, 255), :ok <- validate_fanout(fanout), :ok <- validate_count(count), {:ok, shas, rest} <- parse_shas(rest, count), :ok <- validate_shas_sorted(shas), {:ok, _crcs, rest} <- parse_crcs(rest, count), {:ok, offsets_4, rest} <- parse_offsets_4(rest, count), {:ok, large_offsets} <- parse_large_offsets(rest, offsets_4), {:ok, offset_map} <- build_offset_map(shas, offsets_4, large_offsets) do {:ok, %__MODULE__{ fanout: fanout, shas: shas, offsets: offset_map, count: count }} end end end
def parse(_), do: {:error, :invalid_idx_header}
# Verify the trailing 20-byte index checksum defp verify_index_checksum(idx_data) when byte_size(idx_data) >= 28 do body_len = byte_size(idx_data) - 20 <<body::binary-size(body_len), checksum::binary-size(20)>> = idx_data expected = :crypto.hash(:sha, body)
if checksum == expected do :ok else {:error, :index_checksum_mismatch} end end
defp verify_index_checksum(_), do: {:error, :index_too_short}
# Validate that SHA list is sorted (PK7) defp validate_shas_sorted([]), do: :ok defp validate_shas_sorted([_]), do: :ok
defp validate_shas_sorted(shas) do sorted? = shas |> Enum.chunk_every(2, 1, :discard) |> Enum.all?(fn [a, b] -> a < b end)
if sorted?, do: :ok, else: {:error, :shas_not_sorted} end
@doc """ Look up the pack offset for a given SHA.
|
|
|
lib/ex_git_objectstore/pack/reader.ex
|
+165
−22
|
@@ -32,6 +32,7 @@
@pack_signature "PACK" @max_pack_objects 10_000_000 @max_delta_depth 50
# Object types (3-bit) @obj_commit 1 @@ -52,14 +53,30 @@ Returns a list of `%{type: atom, data: binary, offset: integer}`. """ @spec parse(binary()) :: {:ok, [pack_object()]} | {:error, term()} def parse(<<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, rest::binary>>) when version == 2 do def parse( <<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, _rest::binary>> = pack_data ) when version in [2, 3] do if count > @max_pack_objects do {:error, {:pack_too_large, count}} else # Verify trailing checksum (PK1) case verify_pack_checksum(pack_data) do :ok -> # Strip the trailing 20-byte checksum before parsing entries entries_len = byte_size(pack_data) - 12 - 20
<<_header::binary-size(12), entries_data::binary-size(entries_len), _checksum::binary-size(20)>> = pack_data
case parse_entries(entries_data, count, [], %{}) do {:ok, entries, _cache} -> {:ok, entries} {:error, _} = err -> err end
{:error, _} = err -> err case parse_entries(rest, count, [], %{}) do {:ok, entries, _cache} -> {:ok, entries} {:error, _} = err -> err end end end @@ -67,6 +84,24 @@ def parse(_), do: {:error, :invalid_pack_header}
@doc """ Verify the trailing SHA-1 checksum of a pack file. """ @spec verify_pack_checksum(binary()) :: :ok | {:error, term()} def verify_pack_checksum(pack_data) when byte_size(pack_data) >= 32 do body_len = byte_size(pack_data) - 20 <<body::binary-size(body_len), checksum::binary-size(20)>> = pack_data expected = :crypto.hash(:sha, body)
if checksum == expected do :ok else {:error, :pack_checksum_mismatch} end end
def verify_pack_checksum(_), do: {:error, :pack_too_short}
@doc """ Read a single object at the given offset from a packfile binary. Resolves deltas recursively using the pack data itself. """ @@ -81,13 +116,7 @@ @spec read_object(binary(), non_neg_integer(), map()) :: {:ok, {atom(), binary()}} | {:error, term()} def read_object(pack_data, offset, cache) do case Map.get(cache, offset) do {type, data} -> {:ok, {type, data}}
nil -> do_read_object(pack_data, offset, cache) end do_read_object(pack_data, offset, cache, 0) end
# -- Private -- @@ -169,7 +198,21 @@ end end
defp do_read_object(pack_data, offset, cache) do defp do_read_object(_pack_data, _offset, _cache, depth) when depth > @max_delta_depth do {:error, :max_delta_depth_exceeded} end
defp do_read_object(pack_data, offset, cache, depth) do case Map.get(cache, offset) do {type, data} -> {:ok, {type, data}}
nil -> resolve_object(pack_data, offset, cache, depth) end end
defp resolve_object(pack_data, offset, cache, depth) do <<_skip::binary-size(offset), data::binary>> = pack_data
case parse_object_header(data) do @@ -197,7 +240,7 @@
with {:ok, delta_data, _} <- decompress_data(compressed), {:ok, {base_type, base_data}} <- do_read_object(pack_data, base_offset, cache, depth + 1), read_object(pack_data, base_offset, cache), {:ok, result} <- Delta.apply(base_data, delta_data) do {:ok, {base_type, result}} end @@ -215,7 +258,7 @@ with {:ok, delta_data, _} <- decompress_data(compressed), {:ok, base_offset} <- find_sha_offset(pack_data, base_sha, cache), {:ok, {base_type, base_data}} <- read_object(pack_data, base_offset, cache), do_read_object(pack_data, base_offset, cache, depth + 1), {:ok, result} <- Delta.apply(base_data, delta_data) do {:ok, {base_type, result}} end @@ -352,12 +395,112 @@ defp type_num_to_atom(@obj_blob), do: :blob defp type_num_to_atom(@obj_tag), do: :tag
# Build an in-memory SHA-to-offset index by scanning pack entries. # For non-delta objects, compute the SHA from type+content. # This enables REF_DELTA resolution without a .idx file. defp find_sha_offset(pack_data, sha, cache) do # Check cache first (caller may have provided from .idx) case Map.get(cache, {:sha, sha}) do offset when is_integer(offset) -> {:ok, offset}
nil -> # Build index by scanning the pack case build_sha_index(pack_data) do {:ok, index} -> case Map.get(index, sha) do nil -> {:error, {:ref_delta_base_not_found, sha}} offset -> {:ok, offset} end
{:error, _} = err -> err end # TODO: Implement REF_DELTA resolution by building an in-memory index from # the pack data. Currently REF_DELTA objects can only be resolved when the # caller provides offsets via the cache (e.g., from a .idx file lookup). # Without this, thin packs from git-fetch that use REF_DELTA will fail. # See: https://git-scm.com/docs/pack-format#_deltified_representation defp find_sha_offset(_pack_data, _sha, _cache) do {:error, :ref_delta_not_supported} end end
@doc """ Build a SHA-to-offset index by scanning all non-delta entries in a pack. """ @spec build_sha_index(binary()) :: {:ok, map()} | {:error, term()} def build_sha_index( <<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, rest::binary>> ) when version in [2, 3] do scan_entries_for_index(rest, count, 12, %{}) end
def build_sha_index(_), do: {:error, :invalid_pack_header}
defp scan_entries_for_index(_data, 0, _offset, index), do: {:ok, index}
defp scan_entries_for_index(data, remaining, offset, index) do case parse_object_header(data) do {:ok, type_num, _size, header_len, rest_after_header} -> case type_num do t when t in [@obj_commit, @obj_tree, @obj_blob, @obj_tag] -> type_str = Atom.to_string(type_num_to_atom(t))
case decompress_data(rest_after_header) do {:ok, content, compressed_len} -> # Compute SHA: hash("type size\0content") raw = "#{type_str} #{byte_size(content)}\0" <> content sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower) consumed = header_len + compressed_len <<_::binary-size(consumed), next_data::binary>> = data
scan_entries_for_index( next_data, remaining - 1, offset + consumed, Map.put(index, sha, offset) )
{:error, _} = err -> err end
@obj_ofs_delta -> case parse_ofs_delta_offset(rest_after_header) do {:ok, _neg_offset, ofs_len, rest_after_ofs} -> case decompress_data(rest_after_ofs) do {:ok, _delta_data, compressed_len} -> consumed = header_len + ofs_len + compressed_len <<_::binary-size(consumed), next_data::binary>> = data # We don't index delta objects (they need resolution first) scan_entries_for_index(next_data, remaining - 1, offset + consumed, index)
{:error, _} = err -> err end
{:error, _} = err -> err end
@obj_ref_delta -> if byte_size(rest_after_header) >= 20 do <<_base_sha::binary-size(20), rest_after_sha::binary>> = rest_after_header
case decompress_data(rest_after_sha) do {:ok, _delta_data, compressed_len} -> consumed = header_len + 20 + compressed_len <<_::binary-size(consumed), next_data::binary>> = data scan_entries_for_index(next_data, remaining - 1, offset + consumed, index)
{:error, _} = err -> err end else {:error, :truncated_ref_delta} end
_ -> {:error, {:unknown_object_type, type_num}} end
{:error, _} = err -> err end end end
|
|
|
lib/ex_git_objectstore/pack/writer.ex
|
+14
−10
|
@@ -180,19 +180,23 @@ end
defp build_fanout(sorted_entries) do # 256 entries, each is cumulative count of objects with first SHA byte <= N sha_first_bytes = Enum.map(sorted_entries, fn {sha, _offset, _crc} -> # O(N) fanout computation: count objects per bucket, then cumulative sum counts = :array.new(256, default: 0)
counts = Enum.reduce(sorted_entries, counts, fn {sha, _offset, _crc}, acc -> <<first_byte, _rest::binary>> = elem(Base.decode16(sha, case: :mixed), 1) first_byte :array.set(first_byte, :array.get(first_byte, acc) + 1, acc) end)
fanout = for i <- 0..255 do Enum.count(sha_first_bytes, fn b -> b <= i end) end # Build cumulative fanout {fanout_list, _} = Enum.reduce(0..255, {[], 0}, fn i, {acc, cumulative} -> total = cumulative + :array.get(i, counts) {[<<total::unsigned-big-32>> | acc], total} end)
fanout |> Enum.map(fn count -> <<count::unsigned-big-32>> end) fanout_list |> Enum.reverse() |> IO.iodata_to_binary() end
|
|
|
lib/ex_git_objectstore/protocol/pkt_line.ex
|
+30
−5
|
@@ -23,14 +23,18 @@ - `NNNN<data>` — data packet where NNNN is total length (including the 4 prefix bytes)
Minimum data packet length is `0005` (4 prefix + 1 data byte). Maximum is `ffff` (65535 bytes total, 65531 data bytes). Maximum is `fff0` (65520 bytes total, 65516 data bytes). """
@flush "0000" @delim "0001" @response_end "0002"
# Max total pkt-line is 65520 (0xFFF0) bytes per spec. # Max payload = 65520 - 4 (length prefix) = 65516 @max_pkt_len 65520 @max_data_len 65520 # For sideband: max payload (65516) - 1 band byte = 65515 @max_sideband_data 65515
@doc """ Encode a single data line as a pkt-line. @@ -40,6 +44,11 @@ def encode(data) when is_binary(data) do data = if String.ends_with?(data, "\n"), do: data, else: data <> "\n" len = byte_size(data) + 4
if len > @max_pkt_len do raise ArgumentError, "pkt-line too long: #{len} bytes (max #{@max_pkt_len})" end
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0") <<hex_len::binary, data::binary>> end @@ -50,6 +59,11 @@ @spec encode_raw(binary()) :: binary() def encode_raw(data) when is_binary(data) do len = byte_size(data) + 4
if len > @max_pkt_len do raise ArgumentError, "pkt-line too long: #{len} bytes (max #{@max_pkt_len})" end
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0") <<hex_len::binary, data::binary>> end @@ -109,8 +123,8 @@ {len, ""} when len >= 4 -> if byte_size(data) >= len do <<_hex::binary-size(4), payload::binary-size(len - 4), rest::binary>> = data # Strip trailing newline if present # Strip exactly one trailing LF if present (spec says receivers should strip it) payload = strip_one_trailing_lf(payload) payload = String.trim_trailing(payload, "\n") {:ok, {:data, payload}, rest} else {:need_more, data} @@ -143,7 +157,7 @@ """ @spec encode_sideband(1 | 2 | 3, binary()) :: iodata() def encode_sideband(band, data) when band in [1, 2, 3] do chunks = chunk_binary(data, @max_sideband_data) chunks = chunk_binary(data, @max_data_len)
Enum.map(chunks, fn chunk -> encode_raw(<<band, chunk::binary>>) @@ -167,5 +181,16 @@
{:error, _} = err -> err end end
defp strip_one_trailing_lf(<<>>), do: <<>>
defp strip_one_trailing_lf(bin) do size = byte_size(bin) - 1
case bin do <<prefix::binary-size(size), "\n">> -> prefix _ -> bin end end
|
|
|
lib/ex_git_objectstore/protocol/receive_pack.ex
|
+145
−46
|
@@ -27,12 +27,15 @@ in and reads response data out. """
alias ExGitObjectstore.{Object, Ref, Repo} alias ExGitObjectstore.{Object, ObjectResolver, Ref, Repo} alias ExGitObjectstore.Pack.Reader alias ExGitObjectstore.Protocol.PktLine
@zero_sha String.duplicate("0", 40)
# Maximum pack buffer size (256 MB) @max_pack_size 256 * 1024 * 1024
# Capabilities we actually implement. Others are scaffolded but not yet functional: # TODO: ofs-delta — Writer doesn't generate OFS_DELTA yet # TODO: side-band-64k — not yet used in report responses @@ -48,14 +51,16 @@ repo: Repo.t(), phase: :advertise | :commands | :pack | :done, commands: [command()], client_caps: MapSet.t(), pack_buffer: binary(), result: :ok | {:error, term()} result: nil | :ok | {:error, term()} }
defstruct [ :repo, phase: :advertise, commands: [], client_caps: MapSet.new(), pack_buffer: <<>>, result: nil ] @@ -81,7 +86,14 @@ @spec feed(state(), binary()) :: {binary(), state()} def feed(%__MODULE__{phase: :commands} = state, data) do case parse_commands(data) do {:ok, commands, rest} -> state = %{state | commands: commands, phase: :pack, pack_buffer: rest} {:ok, commands, client_caps, rest} -> state = %{ state | commands: commands, client_caps: client_caps, phase: :pack, pack_buffer: rest }
# Check if the pack is already complete in the buffer maybe_process_pack(state) @@ -97,8 +109,15 @@ end
def feed(%__MODULE__{phase: :pack} = state, data) do state = %{state | pack_buffer: state.pack_buffer <> data} maybe_process_pack(state) new_buffer = state.pack_buffer <> data
if byte_size(new_buffer) > @max_pack_size do report = build_error_report(:pack_too_large, state.commands) {report, %{state | phase: :done, result: {:error, :pack_too_large}}} else state = %{state | pack_buffer: new_buffer} maybe_process_pack(state) end end
def feed(%__MODULE__{phase: :done} = state, _data) do @@ -115,20 +134,20 @@ # -- Private --
defp build_advertisement(repo) do refs = list_all_refs_with_head(repo) refs = list_all_refs(repo)
case refs do [] -> # Empty repo — advertise capabilities with zero SHA line = "#{@zero_sha} capabilities^{}\0 #{@capabilities}" line = "#{@zero_sha} capabilities^{}\0#{@capabilities}" PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] -> first_line = "#{first_sha} #{first_ref}\0 #{@capabilities}" first_line = "#{first_sha} #{first_ref}\0#{@capabilities}"
lines = [PktLine.encode(first_line)] ++ Enum.map(rest, fn {ref, sha} -> PktLine.encode("#{sha} #{ref}") Enum.flat_map(rest, fn {ref, sha} -> [PktLine.encode("#{sha} #{ref}")] end) ++ [PktLine.flush()] @@ -137,7 +156,9 @@ end end
defp list_all_refs(repo) do # Build the ref list with HEAD first (spec: HEAD MUST appear first). # Also includes peeled tag refs (^{}) for annotated tags. defp list_all_refs_with_head(repo) do heads = case Ref.list(repo, "refs/heads/") do {:ok, refs} -> refs @@ -150,28 +171,76 @@ _ -> [] end
(heads ++ tags) |> Enum.sort_by(fn {ref, _sha} -> ref end) sorted = (heads ++ tags) |> Enum.sort_by(fn {ref, _sha} -> ref end)
# Add peeled tag refs sorted_with_peeled = Enum.flat_map(sorted, fn {ref, sha} = entry -> if String.starts_with?(ref, "refs/tags/") do case peel_tag(repo, sha) do {:ok, peeled_sha} when peeled_sha != sha -> [entry, {"#{ref}^{}", peeled_sha}]
_ -> [entry] end else [entry] end end)
# Resolve HEAD and prepend it case resolve_head_sha(repo) do {:ok, head_sha} -> [{"HEAD", head_sha} | sorted_with_peeled]
_ -> sorted_with_peeled end end
defp resolve_head_sha(repo) do case Ref.get_head(repo) do {:ok, "ref: " <> ref_path} -> Ref.get(repo, ref_path) {:ok, sha} -> {:ok, sha} error -> error end end
defp peel_tag(repo, sha) do case ObjectResolver.read(repo, sha) do {:ok, %ExGitObjectstore.Object.Tag{object: target_sha}} -> {:ok, target_sha}
_ -> {:ok, sha} end end
defp parse_commands(data) do parse_command_lines(data, []) parse_command_lines(data, [], nil) end
defp parse_command_lines(data, acc, caps) do defp parse_command_lines(data, acc) do case PktLine.decode_one(data) do {:ok, :flush, rest} -> client_caps = caps || MapSet.new()
if acc == [] do # No commands — client is just probing {:ok, [], rest} {:ok, [], client_caps, rest} else {:ok, Enum.reverse(acc), rest} {:ok, Enum.reverse(acc), client_caps, rest} end
{:ok, {:data, line}, rest} -> case parse_command_line(line) do {:ok, cmd} -> parse_command_lines(rest, [cmd | acc]) {:ok, cmd, line_caps} -> # Only the first command line carries capabilities caps = caps || line_caps parse_command_lines(rest, [cmd | acc], caps)
{:error, _} = err -> err @@ -187,18 +256,26 @@
defp parse_command_line(line) do # Format: "<old-sha> <new-sha> <ref-name>[\0<capabilities>]" # Strip capabilities if present line = # Extract capabilities if present {cmd_str, caps} = case String.split(line, "\0", parts: 2) do [cmd, _caps] -> cmd [cmd] -> cmd [cmd, caps_str] -> cap_set = caps_str |> String.split(" ", trim: true) |> MapSet.new()
{cmd, cap_set}
[cmd] -> {cmd, MapSet.new()} end
case String.split(line, " ", parts: 3) do case String.split(cmd_str, " ", parts: 3) do [old_sha, new_sha, ref] when byte_size(old_sha) == 40 and byte_size(new_sha) == 40 -> {:ok, %{ref: ref, old_sha: old_sha, new_sha: new_sha}, caps} {:ok, %{ref: ref, old_sha: old_sha, new_sha: new_sha}}
_ -> {:error, {:invalid_command, line}} {:error, {:invalid_command, cmd_str}} end end @@ -217,10 +294,6 @@
:incomplete -> {<<>>, state}
{:error, reason} -> report = build_error_report(reason) {report, %{state | phase: :done, result: {:error, reason}}} end end end @@ -231,7 +304,7 @@ # A complete pack has: header (12 bytes) + entries + 20-byte checksum # We can't easily know the total size without parsing all entries, # so we verify the trailing SHA-1 checksum if byte_size(data) > 32 do if byte_size(data) >= 32 do body_len = byte_size(data) - 20 <<body::binary-size(body_len), checksum::binary-size(20)>> = data expected = :crypto.hash(:sha, body) @@ -256,7 +329,7 @@ process_ref_updates(state)
{:error, reason} -> report = build_error_report(reason) report = build_error_report(reason, state.commands) {report, %{state | phase: :done, result: {:error, reason}}} end end @@ -267,10 +340,15 @@ Enum.reduce_while(entries, :ok, fn entry, :ok -> raw = Object.encode_raw_from_type(entry.type, entry.data) sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
case zlib_compress(raw) do {:ok, compressed} -> case Repo.storage_call(repo, :put_object, [sha, compressed]) do :ok -> {:cont, :ok} compressed = zlib_compress(raw) {:error, _} = err -> {:halt, err} end
case Repo.storage_call(repo, :put_object, [sha, compressed]) do :ok -> {:cont, :ok} {:error, _} = err -> {:halt, err} {:error, _} = err -> {:halt, err} end end) @@ -287,8 +365,17 @@ {cmd.ref, result} end)
report = build_report(results) {report, %{state | phase: :done, result: :ok}} # Track whether any ref updates failed any_errors? = Enum.any?(results, fn {_ref, r} -> match?({:error, _}, r) end) overall_result = if any_errors?, do: {:error, :some_refs_failed}, else: :ok
# Only send report-status if client requested it (or if no commands/caps parsed) if MapSet.member?(state.client_caps, "report-status") or MapSet.size(state.client_caps) == 0 do report = build_report(results) {report, %{state | phase: :done, result: overall_result}} else {<<>>, %{state | phase: :done, result: overall_result}} end end
defp apply_ref_command(repo, %{old_sha: @zero_sha, new_sha: new_sha, ref: ref}) do @@ -314,21 +401,33 @@ PktLine.encode("ok #{ref}")
{ref, {:error, reason}} -> PktLine.encode("ng #{ref} #{inspect(reason)}") PktLine.encode("ng #{ref} #{sanitize_error(reason)}") end) ++ [PktLine.flush()]
IO.iodata_to_binary(lines) end
defp build_error_report(reason, commands \\ []) do defp build_error_report(reason) do error_msg = sanitize_error(reason)
ref_lines = Enum.map(commands, fn cmd -> PktLine.encode("ng #{cmd.ref} #{error_msg}") end)
lines = [PktLine.encode("unpack #{error_msg}")] ++ ref_lines ++ [PktLine.flush()] lines = [ PktLine.encode("unpack #{inspect(reason)}"), PktLine.flush() ]
IO.iodata_to_binary(lines) end
defp sanitize_error(reason) when is_atom(reason), do: Atom.to_string(reason) defp sanitize_error(reason) when is_binary(reason), do: reason defp sanitize_error({tag, _details}) when is_atom(tag), do: Atom.to_string(tag) defp sanitize_error(_), do: "internal_error"
defp zlib_compress(data) do z = :zlib.open() @@ -337,7 +436,7 @@ :zlib.deflateInit(z) compressed = :zlib.deflate(z, data, :finish) :zlib.deflateEnd(z) IO.iodata_to_binary(compressed) {:ok, IO.iodata_to_binary(compressed)} rescue e -> {:error, {:compress_failed, Exception.message(e)}} after
|
|
|
lib/ex_git_objectstore/protocol/upload_pack.ex
|
+192
−59
|
@@ -75,11 +75,20 @@ generate_pack_response(state)
{:ok, wants, haves, :continue} -> # Client wants more negotiation rounds — simplified: just ACK and continue state = %{state | wants: wants ++ state.wants, haves: haves ++ state.haves} nak = PktLine.encode("NAK") {nak, state}
# Find common objects: ACK the last have that we have in our repo case find_common(state.repo, haves) do {:ok, common_sha} -> ack = PktLine.encode("ACK #{common_sha}") state = %{state | common: [common_sha | state.common]} {ack, state}
:none -> nak = PktLine.encode("NAK") {nak, state} end
{:error, _reason} -> nak = PktLine.encode("NAK") {nak, %{state | phase: :done}} @@ -100,21 +109,21 @@ # -- Private --
defp build_advertisement(repo) do refs = list_all_refs(repo) refs = list_all_refs_with_head(repo) symref = head_symref(repo) caps = build_capabilities(symref)
case refs do [] -> line = "#{@zero_sha} capabilities^{}\0 #{caps}" line = "#{@zero_sha} capabilities^{}\0#{caps}" PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] -> first_line = "#{first_sha} #{first_ref}\0 #{caps}" first_line = "#{first_sha} #{first_ref}\0#{caps}"
lines = [PktLine.encode(first_line)] ++ Enum.map(rest, fn {ref, sha} -> PktLine.encode("#{sha} #{ref}") Enum.flat_map(rest, fn {ref, sha} -> [PktLine.encode("#{sha} #{ref}")] end) ++ [PktLine.flush()] @@ -133,7 +142,9 @@ defp build_capabilities(nil), do: @capabilities defp build_capabilities(ref), do: "#{@capabilities} symref=HEAD:#{ref}"
defp list_all_refs(repo) do # Build the ref list with HEAD first (spec: HEAD MUST appear first). # Also includes peeled tag refs (^{}) for annotated tags. defp list_all_refs_with_head(repo) do heads = case Ref.list(repo, "refs/heads/") do {:ok, refs} -> refs @@ -146,43 +157,140 @@ _ -> [] end
sorted = (heads ++ tags) |> Enum.sort_by(fn {ref, _sha} -> ref end)
# Add peeled tag refs: for annotated tags, add a refname^{} entry # showing the commit the tag points to sorted_with_peeled = Enum.flat_map(sorted, fn {ref, sha} = entry -> if String.starts_with?(ref, "refs/tags/") do case peel_tag(repo, sha) do {:ok, peeled_sha} when peeled_sha != sha -> [entry, {"#{ref}^{}", peeled_sha}]
_ -> [entry] end else [entry] end end)
# Resolve HEAD and prepend it case resolve_head_sha(repo) do {:ok, head_sha} -> [{"HEAD", head_sha} | sorted_with_peeled]
_ -> (heads ++ tags) |> Enum.sort_by(fn {ref, _sha} -> ref end) sorted_with_peeled end end
defp resolve_head_sha(repo) do case Ref.get_head(repo) do {:ok, "ref: " <> ref_path} -> Ref.get(repo, ref_path) {:ok, sha} -> {:ok, sha} error -> error end end
# Peel an annotated tag to its target object SHA. defp peel_tag(repo, sha) do case ObjectResolver.read(repo, sha) do {:ok, %ExGitObjectstore.Object.Tag{object: target_sha}} -> {:ok, target_sha}
_ -> {:ok, sha} end end
defp parse_wants_haves(data) do # Phase 1: Parse want lines until flush case parse_want_lines(data, []) do {:ok, wants, rest} -> # Phase 2: Parse have lines until "done" case parse_have_lines(rest, []) do {:ok, haves, :done} -> {:ok, wants, haves, :done}
{:ok, haves, :continue} -> {:ok, wants, haves, :continue}
{:error, _} = err -> err end
{:error, _} = err -> parse_wh_lines(data, [], [], nil) err end end
defp parse_wh_lines(data, wants, haves, _terminator) do defp parse_want_lines(data, acc) do case PktLine.decode_one(data) do {:ok, :flush, rest} -> {:ok, Enum.reverse(acc), rest}
{:ok, {:data, "want " <> sha_and_caps}, rest} -> sha = sha_and_caps |> String.split(" ", parts: 2) |> List.first() |> String.trim() parse_want_lines(rest, [sha | acc])
{:ok, {:data, "done"}, _rest} -> # Client sent done without any haves (simple clone) {:ok, Enum.reverse(acc), <<>>}
{:ok, {:data, _other}, rest} -> parse_want_lines(rest, acc)
{:need_more, _} -> {:ok, Enum.reverse(acc), <<>>}
{:error, _} = err -> err end end
defp parse_have_lines(<<>>, acc) do # No have data — this is a clone (wants only, no haves) {:ok, Enum.reverse(acc), :done} end {:ok, :flush, _rest} -> {:ok, Enum.reverse(wants), Enum.reverse(haves), :done}
defp parse_have_lines(data, acc) do case PktLine.decode_one(data) do {:ok, {:data, "done"}, _rest} -> {:ok, Enum.reverse(wants), Enum.reverse(haves), :done} {:ok, Enum.reverse(acc), :done}
{:ok, {:data, "want " <> sha_and_caps}, rest} -> # First want line may have capabilities after SHA sha = sha_and_caps |> String.split(" ", parts: 2) |> List.first() |> String.trim() parse_wh_lines(rest, [sha | wants], haves, nil) {:ok, :flush, _rest} -> {:ok, Enum.reverse(acc), :continue}
{:ok, {:data, "have " <> sha}, rest} -> sha = String.trim(sha) parse_wh_lines(rest, wants, [sha | haves], nil) parse_have_lines(rest, [sha | acc])
{:ok, {:data, _other}, rest} -> # Skip unknown lines parse_wh_lines(rest, wants, haves, nil) parse_have_lines(rest, acc)
{:need_more, _} -> {:ok, Enum.reverse(wants), Enum.reverse(haves), :continue} {:ok, Enum.reverse(acc), :continue}
{:error, _} = err -> err end end
# Check if any of the given SHAs exist in our repo. # Returns {:ok, sha} for the last one found, or :none. defp find_common(repo, haves) do Enum.reduce(haves, :none, fn sha, acc -> case ObjectResolver.read(repo, sha) do {:ok, _} -> {:ok, sha} _ -> acc end end) end
defp generate_pack_response(state) do # Collect all objects needed for the wants, excluding objects reachable from haves case collect_objects(state.repo, state.wants, state.haves) do @@ -202,22 +310,28 @@ {response, %{state | phase: :done}}
{:error, reason} -> nak = PktLine.encode("ERR #{inspect(reason)}") nak = PktLine.encode("ERR #{sanitize_error(reason)}") {nak, %{state | phase: :done}} end end
# Defensive: handle various error shapes even though current callers only pass binary @dialyzer {:nowarn_function, sanitize_error: 1} defp sanitize_error(reason) when is_binary(reason), do: reason defp sanitize_error(reason) when is_atom(reason), do: Atom.to_string(reason) defp sanitize_error({tag, _details}) when is_atom(tag), do: Atom.to_string(tag) defp sanitize_error(_), do: "internal_error"
defp collect_objects(repo, wants, haves) do # Walk from each want SHA and collect all reachable objects have_set = MapSet.new(haves)
try do objects = wants |> Enum.flat_map(fn sha -> collect_reachable(repo, sha, have_set) {objects, _visited} = Enum.reduce(wants, {[], MapSet.new()}, fn sha, {acc, visited} -> {new_objects, visited} = collect_reachable(repo, sha, have_set, visited) {acc ++ new_objects, visited} end) |> Enum.uniq_by(fn {_type, _data, sha} -> sha end)
{:ok, objects} rescue @@ -225,64 +339,83 @@ end end
defp collect_reachable(repo, sha, exclude_set) do if MapSet.member?(exclude_set, sha) do [] defp collect_reachable(repo, sha, exclude_set, visited) do if MapSet.member?(exclude_set, sha) or MapSet.member?(visited, sha) do {[], visited} else visited = MapSet.put(visited, sha)
case ObjectResolver.read(repo, sha) do {:ok, %Commit{} = commit} -> # Collect the tree and all referenced objects tree_objects = collect_tree_objects(repo, commit.tree) {tree_objects, visited} = collect_tree_objects(repo, commit.tree, visited)
# Recurse into parents (but stop at haves) {parent_objects, visited} = Enum.reduce(commit.parents, {[], visited}, fn parent_sha, {acc, vis} -> parent_objects = {objs, vis} = collect_reachable(repo, parent_sha, exclude_set, vis) {acc ++ objs, vis} Enum.flat_map(commit.parents, fn parent_sha -> collect_reachable(repo, parent_sha, exclude_set) end)
[{:commit, Object.encode_content_only(commit), sha} | tree_objects ++ parent_objects] objects = [ {:commit, Object.encode_content_only(commit), sha} | tree_objects ++ parent_objects ]
{objects, visited}
{:ok, %Tree{} = tree} -> collect_tree_entry_objects(repo, tree, sha) collect_tree_entry_objects(repo, tree, sha, visited)
{:ok, %Blob{content: content}} -> [{:blob, content, sha}] {[{:blob, content, sha}], visited}
{:error, _} -> [] {[], visited} end end end
defp collect_tree_objects(repo, tree_sha, visited) do if MapSet.member?(visited, tree_sha) do defp collect_tree_objects(repo, tree_sha) do case ObjectResolver.read(repo, tree_sha) do {:ok, %Tree{} = tree} -> {[], visited} else case ObjectResolver.read(repo, tree_sha) do {:ok, %Tree{} = tree} -> collect_tree_entry_objects(repo, tree, tree_sha, visited) collect_tree_entry_objects(repo, tree, tree_sha)
_ -> {[], visited} end _ -> [] end end
defp collect_tree_entry_objects(repo, %Tree{} = tree, tree_sha) do defp collect_tree_entry_objects(repo, %Tree{} = tree, tree_sha, visited) do visited = MapSet.put(visited, tree_sha) tree_content = Tree.encode_content(tree) entry = [{:tree, tree_content, tree_sha}]
child_objects = Enum.flat_map(tree.entries, fn %{mode: "40000", sha: sha} -> {child_objects, visited} = Enum.reduce(tree.entries, {[], visited}, fn %{mode: "40000", sha: sha}, {acc, vis} -> {objs, vis} = collect_tree_objects(repo, sha, vis) {acc ++ objs, vis}
%{sha: sha}, {acc, vis} -> if MapSet.member?(vis, sha) do {acc, vis} else vis = MapSet.put(vis, sha) collect_tree_objects(repo, sha)
%{sha: sha} -> case ObjectResolver.read(repo, sha) do {:ok, %Blob{content: content}} -> [{:blob, content, sha}] case ObjectResolver.read(repo, sha) do {:ok, %Blob{content: content}} -> {acc ++ [{:blob, content, sha}], vis}
_ -> _ -> {acc, vis} end [] end end)
{entry ++ child_objects, visited} entry ++ child_objects end end
|
|
|
lib/ex_git_objectstore/ref.ex
|
+30
−3
|
@@ -24,7 +24,9 @@ """ @spec get(Repo.t(), String.t()) :: {:ok, String.t()} | {:error, term()} def get(%Repo{} = repo, ref_path) do with :ok <- validate_ref_name(ref_path) do Repo.storage_call(repo, :get_ref, [ref_path]) end Repo.storage_call(repo, :get_ref, [ref_path]) end
@doc """ @@ -44,7 +46,9 @@ """ @spec delete(Repo.t(), String.t()) :: :ok | {:error, term()} def delete(%Repo{} = repo, ref_path) do with :ok <- validate_ref_name(ref_path) do Repo.storage_call(repo, :delete_ref, [ref_path]) Repo.storage_call(repo, :delete_ref, [ref_path]) end end
@doc """ @@ -52,7 +56,9 @@ """ @spec list(Repo.t(), String.t()) :: {:ok, [{String.t(), String.t()}]} | {:error, term()} def list(%Repo{} = repo, ref_prefix) do with :ok <- validate_ref_prefix(ref_prefix) do Repo.storage_call(repo, :list_refs, [ref_prefix]) Repo.storage_call(repo, :list_refs, [ref_prefix]) end end
@doc """ @@ -157,6 +163,27 @@
String.starts_with?(ref_path, "/") -> {:error, {:invalid_ref_name, ref_path, :leading_slash}}
true -> :ok end end
@doc """ Validate a ref prefix for listing operations. Rejects path traversal and null bytes but allows trailing slashes. """ @spec validate_ref_prefix(String.t()) :: :ok | {:error, term()} def validate_ref_prefix(prefix) do cond do String.contains?(prefix, "..") -> {:error, {:invalid_ref_prefix, prefix, :dotdot}}
String.contains?(prefix, <<0>>) -> {:error, {:invalid_ref_prefix, prefix, :null_byte}}
String.starts_with?(prefix, "/") -> {:error, {:invalid_ref_prefix, prefix, :leading_slash}}
true -> :ok
|
|
|
lib/ex_git_objectstore/storage/filesystem.ex
|
+6
−6
|
@@ -133,7 +133,7 @@
@impl true def get_ref(config, prefix, ref_path) do path = Path.join([config.root, prefix, ref_path]) path = safe_path(config.root, Path.join([prefix, ref_path]))
case File.read(path) do {:ok, data} -> {:ok, String.trim(data)} @@ -144,6 +144,6 @@
@impl true def put_ref(config, prefix, ref_path, new_sha, old_sha) do path = Path.join([config.root, prefix, ref_path]) path = safe_path(config.root, Path.join([prefix, ref_path])) lock_path = path <> ".lock" File.mkdir_p!(Path.dirname(path)) @@ -195,7 +195,7 @@
@impl true def delete_ref(config, prefix, ref_path) do path = safe_path(config.root, Path.join([prefix, ref_path])) path = Path.join([config.root, prefix, ref_path])
case File.rm(path) do :ok -> :ok @@ -206,7 +206,7 @@
@impl true def list_refs(config, prefix, ref_prefix) do base_dir = Path.join([config.root, prefix, ref_prefix]) base_dir = safe_path(config.root, Path.join([prefix, ref_prefix]))
# Start with loose refs loose_refs = @@ -239,7 +239,7 @@
@impl true def get_head(config, prefix) do path = Path.join([config.root, prefix, "HEAD"]) path = safe_path(config.root, Path.join([prefix, "HEAD"]))
case File.read(path) do {:ok, data} -> {:ok, String.trim(data)} @@ -250,6 +250,6 @@
@impl true def put_head(config, prefix, target) do path = safe_path(config.root, Path.join([prefix, "HEAD"])) path = Path.join([config.root, prefix, "HEAD"]) atomic_write(path, target <> "\n") end
|
|
|
lib/ex_git_objectstore/storage/s3.ex
|
+14
−5
|
@@ -121,7 +121,7 @@
@impl true def get_ref(config, prefix, ref_path) do key = "#{prefix}/#{ref_path}" key = safe_key("#{prefix}/#{ref_path}")
case s3_get(config, key) do {:ok, data} -> {:ok, String.trim(data)} @@ -131,7 +131,7 @@
@impl true def put_ref(config, prefix, ref_path, new_sha, old_sha) do key = safe_key("#{prefix}/#{ref_path}") key = "#{prefix}/#{ref_path}"
if old_sha do # CAS: read current value first @@ -156,6 +156,6 @@
@impl true def delete_ref(config, prefix, ref_path) do key = "#{prefix}/#{ref_path}" key = safe_key("#{prefix}/#{ref_path}") s3_delete(config, key) end @@ -208,11 +208,20 @@
defp object_key(prefix, sha) do <<dir::binary-size(2), rest::binary>> = sha safe_key("#{prefix}/objects/#{dir}/#{rest}") "#{prefix}/objects/#{dir}/#{rest}" end
defp pack_key(prefix, pack_sha, ext) do safe_key("#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}") end "#{prefix}/objects/pack/pack-#{pack_sha}.#{ext}"
# Prevent path traversal in S3 keys defp safe_key(key) do if String.contains?(key, "..") do raise ArgumentError, "path traversal detected in S3 key" end
key end
defp s3_get(config, key) do
|
|
|
mix.exs
|
+2
−1
|
@@ -94,7 +94,8 @@ {:hackney, "~> 1.18"}, {:jason, "~> 1.4"}, {:ex_doc, "~> 0.34", only: :dev, runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false} {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false} ] end end
|
|
|
mix.lock
|
+3
−0
|
@@ -1,11 +1,14 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "credo": {:hex, :credo, "1.7.16", "a9f1389d13d19c631cb123c77a813dbf16449a2aebf602f590defa08953309d4", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0562af33756b21f248f066a9119e3890722031b6d199f22e3cf95550e4f1579"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "ex_aws": {:hex, :ex_aws, "2.6.1", "194582c7b09455de8a5ab18a0182e6dd937d53df82be2e63c619d01bddaccdfa", [:mix], [{:configparser_ex, "~> 5.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67842a08c90a1d9a09dbe4ac05754175c7ca253abe4912987c759395d4bd9d26"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"}, "ex_doc": {:hex, :ex_doc, "0.40.1", "67542e4b6dde74811cfd580e2c0149b78010fd13001fda7cfeb2b2c2ffb1344d", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "bcef0e2d360d93ac19f01a85d58f91752d930c0a30e2681145feea6bd3516e00"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
|
|
|
test/ex_git_objectstore/object/blob_test.exs
|
+18
−3
|
@@ -47,10 +47,25 @@ end end
describe "spec compliance - header size validation" do test "decode_raw rejects size mismatch" do # Construct a raw object with wrong size in header # Header says 10 bytes but content is only 5 raw = "blob 10\0hello" result = Object.decode_raw(raw) assert {:error, {:size_mismatch, expected: 10, actual: 5}} = result end
test "decode_raw accepts correct size" do raw = "blob 5\0hello" assert {:ok, %Blob{content: "hello"}} = Object.decode_raw(raw) end end
describe "blob round-trip" do test "encode then decode returns original" do blob = Blob.from_content("test content here") encoded = Object.encode(blob) encoded = Object.encode!(blob) assert {:ok, decoded} = Object.decode(encoded) assert %Blob{content: "test content here"} = decoded end @@ -58,7 +73,7 @@ test "round-trip preserves binary content" do content = <<0, 1, 2, 255, 128, 64>> blob = Blob.from_content(content) encoded = Object.encode!(blob) encoded = Object.encode(blob) assert {:ok, decoded} = Object.decode(encoded) assert decoded.content == content end @@ -66,7 +81,7 @@ test "round-trip preserves large content" do content = String.duplicate("x", 100_000) blob = Blob.from_content(content) encoded = Object.encode(blob) encoded = Object.encode!(blob) assert {:ok, decoded} = Object.decode(encoded) assert decoded.content == content end
|
|
|
test/ex_git_objectstore/object/commit_test.exs
|
+60
−4
|
@@ -30,7 +30,7 @@ message: "Initial commit\n" }
encoded = Object.encode(commit) encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert %Commit{} = decoded assert decoded.tree == @tree_sha @@ -51,7 +51,7 @@ message: "Second commit\n" }
encoded = Object.encode(commit) encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert decoded.parents == [parent_sha] end @@ -70,7 +70,7 @@ message: "Merge commit\n" }
encoded = Object.encode(commit) encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert decoded.parents == parents end @@ -112,6 +112,62 @@ assert content == expected end
test "encode and decode commit with GPG signature (continuation lines)" do gpgsig = "-----BEGIN PGP SIGNATURE-----\n" <> "iQEzBAABCAAdFiEEYp1234567890\n" <> "abcdef1234567890==\n" <> "-----END PGP SIGNATURE-----"
commit = %Commit{ tree: @tree_sha, parents: [], author: "Test User <test@example.com> 1234567890 +0000", committer: "Test User <test@example.com> 1234567890 +0000", gpgsig: gpgsig, message: "Signed commit\n" }
# Encode and verify continuation lines format content = Commit.encode_content(commit) lines = String.split(content, "\n")
# The gpgsig header should use continuation lines (space prefix) gpgsig_idx = Enum.find_index(lines, &String.starts_with?(&1, "gpgsig ")) assert gpgsig_idx != nil
# Lines after the first gpgsig line should start with space (continuation) continuation_lines = lines |> Enum.drop(gpgsig_idx + 1) |> Enum.take_while(&String.starts_with?(&1, " "))
assert length(continuation_lines) >= 2
# Roundtrip: encode then decode preserves the signature encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert decoded.gpgsig != nil # The decoded gpgsig should contain the continuation line prefix assert String.contains?(decoded.gpgsig, "BEGIN PGP SIGNATURE") assert String.contains?(decoded.gpgsig, "END PGP SIGNATURE") end
test "encode and decode commit with extra headers" do commit = %Commit{ tree: @tree_sha, parents: [], author: "Test User <test@example.com> 1234567890 +0000", committer: "Test User <test@example.com> 1234567890 +0000", extra_headers: [{"mergetag", "some-tag-data"}], message: "Merge commit\n" }
encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert [{"mergetag", "some-tag-data"}] = decoded.extra_headers end
test "encode and decode preserves multi-line message" do commit = %Commit{ tree: @tree_sha, @@ -121,7 +177,7 @@ message: "First line\n\nDetailed description\nwith multiple lines\n" }
encoded = Object.encode(commit) encoded = Object.encode!(commit) assert {:ok, decoded} = Object.decode(encoded) assert decoded.message == "First line\n\nDetailed description\nwith multiple lines\n" end
|
|
|
test/ex_git_objectstore/object/tag_test.exs
|
+34
−2
|
@@ -30,7 +30,7 @@ message: "Release v1.0.0\n" }
encoded = Object.encode(tag) encoded = Object.encode!(tag) assert {:ok, decoded} = Object.decode(encoded) assert %Tag{} = decoded assert decoded.object == @commit_sha @@ -77,6 +77,38 @@ assert content == expected end
test "encode and decode tag without tagger (optional tagger)" do tag = %Tag{ object: @commit_sha, type: "commit", tag: "v0.1.0", tagger: nil, message: "Lightweight-style annotated tag\n" }
encoded = Object.encode!(tag) assert {:ok, decoded} = Object.decode(encoded) assert %Tag{} = decoded assert decoded.tagger == nil assert decoded.tag == "v0.1.0" assert decoded.message == "Lightweight-style annotated tag\n" end
test "encode and decode tag with extra headers" do tag = %Tag{ object: @commit_sha, type: "commit", tag: "v1.1.0", tagger: "Test User <test@example.com> 1234567890 +0000", extra_headers: [{"some-header", "some-value"}], message: "Tag with extra headers\n" }
encoded = Object.encode!(tag) assert {:ok, decoded} = Object.decode(encoded) assert [{"some-header", "some-value"}] = decoded.extra_headers end
test "encode and decode tag with multi-line message" do tag = %Tag{ object: @commit_sha, @@ -86,7 +118,7 @@ message: "Major release\n\nBreaking changes:\n- Changed API\n- New format\n" }
encoded = Object.encode(tag) encoded = Object.encode!(tag) assert {:ok, decoded} = Object.decode(encoded)
assert decoded.message ==
|
|
|
test/ex_git_objectstore/object/tree_test.exs
|
+128
−5
|
@@ -25,7 +25,7 @@ %{mode: "100644", name: "hello.txt", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"} ])
encoded = Object.encode(tree) encoded = Object.encode!(tree) assert {:ok, decoded} = Object.decode(encoded) assert %Tree{entries: [entry]} = decoded assert entry.mode == "100644" @@ -41,7 +41,7 @@ %{mode: "40000", name: "subdir", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"} ])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode() assert {:ok, decoded} = tree |> Object.encode() |> Object.decode() # Should be sorted: a.txt, b.txt, subdir assert [first, second, third] = decoded.entries assert first.name == "a.txt" @@ -55,7 +55,7 @@ %{mode: "40000", name: "lib", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"} ])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode() assert {:ok, decoded} = tree |> Object.encode() |> Object.decode() assert [entry] = decoded.entries assert entry.mode == "40000" end @@ -66,7 +66,7 @@ %{mode: "100755", name: "run.sh", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"} ])
assert {:ok, decoded} = tree |> Object.encode() |> Object.decode() assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode() assert [entry] = decoded.entries assert entry.mode == "100755" end @@ -77,7 +77,7 @@ %{mode: "120000", name: "link", sha: "95d09f2b10159347eece71399a7e2e907ea3df4f"} ])
assert {:ok, decoded} = tree |> Object.encode!() |> Object.decode() assert {:ok, decoded} = tree |> Object.encode() |> Object.decode() assert [entry] = decoded.entries assert entry.mode == "120000" end @@ -113,5 +113,128 @@ names = Enum.map(tree.entries, & &1.name) # "aa/" < "ab" < "ac" assert names == ["aa", "ab", "ac"] end end
describe "spec compliance - mode canonicalization" do test "canonicalizes short modes" do tree = Tree.new([ %{mode: "644", name: "a.txt", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ])
assert [%{mode: "100644"}] = tree.entries end
test "canonicalizes padded modes" do tree = Tree.new([ %{mode: "0100644", name: "a.txt", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ])
assert [%{mode: "100644"}] = tree.entries end
test "canonicalizes executable mode" do tree = Tree.new([ %{mode: "755", name: "run.sh", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ])
assert [%{mode: "100755"}] = tree.entries end
test "canonicalizes directory mode" do tree = Tree.new([ %{mode: "040000", name: "dir", sha: "4b825dc642cb6eb9a060e54bf8d69288fbee4904"} ])
assert [%{mode: "40000"}] = tree.entries end
test "canonicalizes gitlink (submodule) mode" do tree = Tree.new([ %{mode: "0160000", name: "submod", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ])
assert [%{mode: "160000"}] = tree.entries end end
describe "spec compliance - entry name validation" do test "rejects empty name" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: "", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "rejects dot entry" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: ".", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "rejects dotdot entry" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: "..", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "rejects .git entry (case-insensitive)" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: ".GIT", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "rejects slash in name" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: "a/b", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "rejects null byte in name" do assert_raise ArgumentError, ~r/invalid tree entry name/, fn -> Tree.new([ %{mode: "100644", name: "a\0b", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ]) end end
test "allows valid names" do tree = Tree.new([ %{ mode: "100644", name: "valid-file.txt", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" }, %{mode: "100644", name: ".gitignore", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}, %{mode: "100644", name: ".gitmodules", sha: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"} ])
assert length(tree.entries) == 3 end end
describe "spec compliance - directory_mode?" do test "identifies directory modes" do assert Tree.directory_mode?("40000") assert Tree.directory_mode?("040000") assert Tree.directory_mode?("0040000") refute Tree.directory_mode?("100644") refute Tree.directory_mode?("100755") refute Tree.directory_mode?("160000") end end
|
|
|
test/ex_git_objectstore/pack/index_test.exs
|
+56
−0
|
@@ -47,6 +47,62 @@ end end
describe "spec compliance - checksum and sorting" do test "generated index has valid checksum" do # Use Writer to generate a pack+index, then verify index parses blob = ExGitObjectstore.Object.Blob.from_content("test\n") sha = ExGitObjectstore.Object.hash(blob)
{_pack_data, idx_data, _pack_sha} = ExGitObjectstore.Pack.Writer.generate_with_index([{:blob, "test\n", sha}])
# Verify the index checksum by checking the last 20 bytes body_len = byte_size(idx_data) - 20 <<body::binary-size(body_len), checksum::binary-size(20)>> = idx_data assert checksum == :crypto.hash(:sha, body)
# Parse should succeed (validates checksum internally) assert {:ok, _index} = Index.parse(idx_data) end
test "rejects index with corrupted checksum" do blob = ExGitObjectstore.Object.Blob.from_content("test\n") sha = ExGitObjectstore.Object.hash(blob)
{_pack_data, idx_data, _pack_sha} = ExGitObjectstore.Pack.Writer.generate_with_index([{:blob, "test\n", sha}])
# Corrupt the trailing checksum body_len = byte_size(idx_data) - 20 <<body::binary-size(body_len), _checksum::binary-size(20)>> = idx_data bad_checksum = :binary.copy(<<0>>, 20) corrupted = <<body::binary, bad_checksum::binary>>
assert {:error, :index_checksum_mismatch} = Index.parse(corrupted) end
test "SHAs in generated index are sorted" do blob1 = ExGitObjectstore.Object.Blob.from_content("one\n") sha1 = ExGitObjectstore.Object.hash(blob1) blob2 = ExGitObjectstore.Object.Blob.from_content("two\n") sha2 = ExGitObjectstore.Object.hash(blob2) blob3 = ExGitObjectstore.Object.Blob.from_content("three\n") sha3 = ExGitObjectstore.Object.hash(blob3)
objects = [ {:blob, "one\n", sha1}, {:blob, "two\n", sha2}, {:blob, "three\n", sha3} ]
{_pack_data, idx_data, _pack_sha} = ExGitObjectstore.Pack.Writer.generate_with_index(objects)
assert {:ok, index} = Index.parse(idx_data) assert index.shas == Enum.sort(index.shas) end end
# -- Helpers --
defp create_packed_repo do
|
|
|
test/ex_git_objectstore/pack/reader_test.exs
|
+74
−0
|
@@ -67,6 +67,80 @@ end end
describe "spec compliance - pack checksum verification" do test "verify_pack_checksum succeeds on valid pack" do content = "hello world\n" blob = ExGitObjectstore.Object.Blob.from_content(content) sha = ExGitObjectstore.Object.hash(blob) {pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
assert :ok = Reader.verify_pack_checksum(pack_data) end
test "verify_pack_checksum detects corruption" do content = "hello world\n" blob = ExGitObjectstore.Object.Blob.from_content(content) sha = ExGitObjectstore.Object.hash(blob) {pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Corrupt one byte in the middle mid = div(byte_size(pack_data), 2) <<before::binary-size(mid), byte, after_bytes::binary>> = pack_data corrupted = <<before::binary, Bitwise.bxor(byte, 0xFF), after_bytes::binary>>
assert {:error, :pack_checksum_mismatch} = Reader.verify_pack_checksum(corrupted) end
test "verify_pack_checksum rejects too-short data" do assert {:error, :pack_too_short} = Reader.verify_pack_checksum(<<1, 2, 3>>) end
test "parse rejects pack with corrupted checksum" do content = "hello world\n" blob = ExGitObjectstore.Object.Blob.from_content(content) sha = ExGitObjectstore.Object.hash(blob) {pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Corrupt the trailing checksum body_len = byte_size(pack_data) - 20 <<body::binary-size(body_len), _checksum::binary-size(20)>> = pack_data bad_checksum = :binary.copy(<<0>>, 20) corrupted = <<body::binary, bad_checksum::binary>>
assert {:error, :pack_checksum_mismatch} = Reader.parse(corrupted) end end
describe "spec compliance - version support" do test "accepts version 2 packs" do content = "test\n" blob = ExGitObjectstore.Object.Blob.from_content(content) sha = ExGitObjectstore.Object.hash(blob) {pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
# Verify it's version 2 <<"PACK", 2::unsigned-big-32, _rest::binary>> = pack_data assert {:ok, _entries} = Reader.parse(pack_data) end end
describe "spec compliance - SHA index building" do test "build_sha_index returns SHA-to-offset map" do content = "hello\n" blob = ExGitObjectstore.Object.Blob.from_content(content) sha = ExGitObjectstore.Object.hash(blob) {pack_data, _} = ExGitObjectstore.Pack.Writer.generate([{:blob, content, sha}])
assert {:ok, index} = Reader.build_sha_index(pack_data) assert Map.has_key?(index, sha) assert is_integer(Map.get(index, sha)) end
test "build_sha_index rejects invalid header" do assert {:error, :invalid_pack_header} = Reader.build_sha_index("not a pack") end end
# -- Helpers --
defp create_packed_repo_with_objects do
|
|
|
test/ex_git_objectstore/protocol/pkt_line_test.exs
|
+57
−0
|
@@ -141,6 +141,63 @@ end end
describe "spec compliance - max length" do test "encode raises on oversized data" do # Max total pkt-line is 65520 bytes (0xFFF0) # encode adds 4-byte prefix + 1-byte newline, so max data is 65515 chars big_data = String.duplicate("x", 65516)
assert_raise ArgumentError, ~r/pkt-line too long/, fn -> PktLine.encode(big_data) end end
test "encode_raw raises on oversized data" do # encode_raw adds 4-byte prefix, so max payload is 65516 bytes big_data = :binary.copy(<<0>>, 65517)
assert_raise ArgumentError, ~r/pkt-line too long/, fn -> PktLine.encode_raw(big_data) end end
test "encode allows data at exact max length" do # Max data + newline + 4 prefix = 65520 # If data already has newline: 65516 bytes data = 65520 total data = String.duplicate("x", 65515) <> "\n" result = PktLine.encode(data) assert byte_size(result) == 65520 end
test "encode_raw allows data at exact max payload" do # 65516 bytes + 4 prefix = 65520 data = :binary.copy(<<0>>, 65516) result = PktLine.encode_raw(data) assert byte_size(result) == 65520 end end
describe "spec compliance - trailing LF stripping" do test "strips exactly one trailing LF" do # Data "hello\n\n" (2 newlines at end) # Encoding: 4 prefix + 7 data = 11 = 000b encoded = "000bhello\n\n" assert {:ok, {:data, "hello\n"}, ""} = PktLine.decode_one(encoded) end
test "does not strip non-LF trailing bytes" do encoded = PktLine.encode_raw("hello") assert {:ok, {:data, "hello"}, ""} = PktLine.decode_one(encoded) end
test "handles empty payload" do # Length 4 means 0 data bytes, but min is actually 5 for data # encode_raw of empty string: "0004" (4 bytes prefix, 0 data = 4 total) encoded = PktLine.encode_raw("") assert {:ok, {:data, ""}, ""} = PktLine.decode_one(encoded) end end
describe "sideband" do test "decode sideband band 1 (pack data)" do assert {:pack, "data"} = PktLine.decode_sideband(<<1, "data">>)
|
|
|
test/ex_git_objectstore/protocol/receive_pack_test.exs
|
+111
−2
|
@@ -70,9 +70,118 @@ _ -> false end)
assert length(data_lines) >= 1 # HEAD first (spec: HEAD MUST appear first), then refs/heads/main assert length(data_lines) >= 2 {:data, first_line} = hd(data_lines) assert String.contains?(first_line, commit_sha) assert String.contains?(first_line, "HEAD")
# refs/heads/main should appear in subsequent lines all_text = Enum.map(data_lines, fn {:data, d} -> d end) |> Enum.join("\n") assert String.contains?(all_text, "refs/heads/main") end end
describe "spec compliance - advertisement format" do test "HEAD appears first in advertisement with refs" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
blob = Blob.from_content("hello\n") {:ok, blob_sha} = Object.write(repo, blob) tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}]) {:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{ tree: tree_sha, parents: [], author: "Test <t@t.com> 1000000000 +0000", committer: "Test <t@t.com> 1000000000 +0000", message: "init\n" }
{:ok, commit_sha} = Object.write(repo, commit) :ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = ReceivePack.init(repo) {:ok, packets, ""} = PktLine.decode(advert) data_lines = for {:data, d} <- packets, do: d
# First line must be HEAD first = hd(data_lines) assert String.contains?(first, "HEAD") assert String.contains?(first, commit_sha) end
test "capabilities immediately follow NUL (no space)" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
{advert, _state} = ReceivePack.init(repo) {:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
[_ref_part, caps_part] = String.split(first_line, "\0", parts: 2) refute String.starts_with?(caps_part, " ") assert String.contains?(caps_part, "report-status") assert String.contains?(caps_part, "delete-refs") end
test "empty repo advertises zero SHA with capabilities" do repo = RepoHelper.memory_repo() assert String.contains?(first_line, "refs/heads/main") ExGitObjectstore.init(repo)
{advert, _state} = ReceivePack.init(repo) {:ok, packets, ""} = PktLine.decode(advert) data_lines = for {:data, d} <- packets, do: d
first = hd(data_lines) zero_sha = String.duplicate("0", 40) assert String.starts_with?(first, zero_sha) assert String.contains?(first, "capabilities^{}") end end
describe "spec compliance - client capabilities" do test "parses client capabilities from first command line" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
blob = Blob.from_content("hello\n") {:ok, blob_sha} = Object.write(repo, blob) tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}]) {:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{ tree: tree_sha, parents: [], author: "Test <t@t.com> 1000000000 +0000", committer: "Test <t@t.com> 1000000000 +0000", message: "init\n" }
{:ok, commit_sha} = Object.write(repo, commit)
{_advert, state} = ReceivePack.init(repo)
# Send command with capabilities after NUL zero = String.duplicate("0", 40) cmd_with_caps = "#{zero} #{commit_sha} refs/heads/main\0report-status delete-refs" commands = PktLine.encode_raw(cmd_with_caps <> "\n") <> PktLine.flush()
objects = [ {:blob, "hello\n", blob_sha}, {:tree, Tree.encode_content(tree), tree_sha}, {:commit, Commit.encode_content(commit), commit_sha} ]
{pack_data, _} = Writer.generate(objects)
{response, state} = ReceivePack.feed(state, commands <> pack_data)
assert ReceivePack.done?(state) # Client sent report-status capability, so we should get a report {:ok, packets, _} = PktLine.decode(response) data_lines = for {:data, d} <- packets, do: d assert "unpack ok" in data_lines end end
|
|
|
test/ex_git_objectstore/protocol/upload_pack_test.exs
|
+61
−0
|
@@ -68,6 +68,67 @@ end end
describe "spec compliance - advertisement format" do test "HEAD appears first in advertisement" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
{commit_sha, _, _} = create_commit(repo, "hello\n", "init\n") :ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = UploadPack.init(repo) {:ok, packets, ""} = PktLine.decode(advert) data_lines = for {:data, d} <- packets, do: d
# First line must be HEAD (spec requirement) first = hd(data_lines) assert String.contains?(first, "HEAD") assert String.contains?(first, commit_sha) end
test "capabilities immediately follow NUL (no space)" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo)
# Parse raw bytes: the first pkt-line should contain \0 followed immediately by caps {:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
# There should be a NUL byte separating ref info from capabilities assert String.contains?(first_line, "\0")
# After NUL, the next character should NOT be a space [_ref_part, caps_part] = String.split(first_line, "\0", parts: 2) refute String.starts_with?(caps_part, " ") end
test "includes symref capability" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo) {:ok, {:data, first_line}, _rest} = PktLine.decode_one(advert)
[_ref_part, caps_part] = String.split(first_line, "\0", parts: 2) assert String.contains?(caps_part, "symref=HEAD:refs/heads/main") end
test "empty repo advertises zero SHA with capabilities" do repo = RepoHelper.memory_repo() ExGitObjectstore.init(repo)
{advert, _state} = UploadPack.init(repo) {:ok, packets, ""} = PktLine.decode(advert) data_lines = for {:data, d} <- packets, do: d
first = hd(data_lines) zero_sha = String.duplicate("0", 40) assert String.starts_with?(first, zero_sha) assert String.contains?(first, "capabilities^{}") end end
describe "feed - clone (wants only, no haves)" do test "sends packfile with all objects for a clone" do repo = RepoHelper.memory_repo()
|
|
|
test/ex_git_objectstore/security_test.exs
|
+255
−0
|
@@ -1,0 +1,255 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.SecurityTest do use ExUnit.Case, async: true
alias ExGitObjectstore.{Object, Ref} alias ExGitObjectstore.Object.{Blob, Commit, Tag, Tree} alias ExGitObjectstore.Pack.Delta alias ExGitObjectstore.Storage.Filesystem alias ExGitObjectstore.Repo alias ExGitObjectstore.Test.RepoHelper
describe "path traversal prevention in filesystem ref operations" do setup do tmp_dir = System.tmp_dir!() root = Path.join(tmp_dir, "sec_test_#{:erlang.unique_integer([:positive])}") File.mkdir_p!(root)
repo = Repo.new("test-repo", storage: {Filesystem, %{root: root}})
on_exit(fn -> File.rm_rf!(root) end) %{repo: repo, root: root} end
test "put_ref rejects path traversal via ref name", %{repo: repo} do sha = String.duplicate("a", 40)
assert {:error, {:invalid_ref_name, _, :dotdot}} = Ref.put(repo, "refs/heads/../../etc/passwd", sha, nil) end
test "get rejects path traversal via ref name", %{repo: repo} do assert {:error, {:invalid_ref_name, _, :dotdot}} = Ref.get(repo, "refs/heads/../../../etc/shadow") end
test "delete rejects path traversal via ref name", %{repo: repo} do assert {:error, {:invalid_ref_name, _, :dotdot}} = Ref.delete(repo, "refs/heads/../../etc/important") end
test "list rejects path traversal via prefix", %{repo: repo} do assert {:error, {:invalid_ref_prefix, _, :dotdot}} = Ref.list(repo, "refs/../../") end
test "filesystem safe_path blocks traversal in ref paths directly", %{root: root} do config = %{root: root}
# Need enough ".." to escape root entirely (root/repos/test/../../../../esc) assert_raise ArgumentError, ~r/path traversal/, fn -> Filesystem.put_ref( config, "repos/test", "../../../../etc/passwd", String.duplicate("a", 40), nil ) end end end
describe "ref name validation on all public functions" do test "get validates ref name" do repo = RepoHelper.memory_repo() assert {:error, {:invalid_ref_name, "", :empty}} = Ref.get(repo, "") end
test "delete validates ref name" do repo = RepoHelper.memory_repo() assert {:error, {:invalid_ref_name, "", :empty}} = Ref.delete(repo, "") end
test "list validates ref prefix" do repo = RepoHelper.memory_repo() assert {:error, {:invalid_ref_prefix, _, :null_byte}} = Ref.list(repo, "refs/\0bad/") end end
describe "decompression size limit" do test "decode rejects oversized decompressed objects" do # Create a large blob and compress it large_content = :binary.copy(<<0>>, 200 * 1024 * 1024) blob = Blob.from_content(large_content)
# This should fail because the decompressed object exceeds the 128MB limit case Object.encode(blob) do {:ok, encoded} -> assert {:error, {:object_too_large, _, _}} = Object.decode(encoded)
{:error, _} -> # Compression itself may fail with very large data, that's ok :ok end end end
describe "encode!/1 convention" do test "encode! raises on failure" do # Normal encoding should work fine blob = Blob.from_content("hello") assert is_binary(Object.encode!(blob)) end
test "encode/1 returns ok/error tuple" do blob = Blob.from_content("hello") assert {:ok, encoded} = Object.encode(blob) assert is_binary(encoded) end end
describe "@enforce_keys on structs" do test "Commit requires tree, author, committer, message via struct!" do assert_raise ArgumentError, ~r/the following keys must also be given/, fn -> struct!(Commit, %{}) end end
test "Tag requires object, type, tag, tagger, message via struct!" do assert_raise ArgumentError, ~r/the following keys must also be given/, fn -> struct!(Tag, %{}) end end
test "Commit with all required keys succeeds" do commit = struct!(Commit, %{ tree: String.duplicate("a", 40), author: "Test <t@t.com> 0 +0000", committer: "Test <t@t.com> 0 +0000", message: "test\n" })
assert commit.tree == String.duplicate("a", 40) end
test "Tag with all required keys succeeds" do tag = struct!(Tag, %{ object: String.duplicate("a", 40), type: "commit", tag: "v1.0", tagger: "Test <t@t.com> 0 +0000", message: "test\n" })
assert tag.tag == "v1.0" end end
describe "tree mode canonicalization" do test "644 is normalized to 100644" do tree = Tree.new([%{mode: "644", name: "f.txt", sha: String.duplicate("a", 40)}]) assert hd(tree.entries).mode == "100644" end
test "755 is normalized to 100755" do tree = Tree.new([%{mode: "755", name: "f.sh", sha: String.duplicate("a", 40)}]) assert hd(tree.entries).mode == "100755" end
test "040000 is normalized to 40000" do tree = Tree.new([%{mode: "040000", name: "dir", sha: String.duplicate("a", 40)}]) assert hd(tree.entries).mode == "40000" end
test "standard modes are unchanged" do sha = String.duplicate("a", 40)
tree = Tree.new([ %{mode: "100644", name: "a.txt", sha: sha}, %{mode: "100755", name: "b.sh", sha: sha}, %{mode: "40000", name: "dir", sha: sha}, %{mode: "120000", name: "link", sha: sha} ])
modes = Enum.map(tree.entries, & &1.mode) assert "100644" in modes assert "100755" in modes assert "40000" in modes assert "120000" in modes end end
describe "delta safety" do test "truncated delta copy command returns error" do base = "hello world" # Build a delta that tries to read copy bytes from an empty buffer # varint for base_size=11, target_size=5, then copy cmd with all offset bits set but no data base_size_varint = <<11>> target_size_varint = <<5>> # Copy command: MSB set, bits 0-3 set (4 offset bytes expected), but no data follows copy_cmd = <<0xFF>>
delta = base_size_varint <> target_size_varint <> copy_cmd assert {:error, :truncated_delta_copy} = Delta.apply(base, delta) end end
describe "receive-pack buffer limit" do test "rejects oversized pack data" do alias ExGitObjectstore.Protocol.ReceivePack alias ExGitObjectstore.Protocol.PktLine
repo = RepoHelper.memory_repo() {_advert, state} = ReceivePack.init(repo)
sha = String.duplicate("a", 40) zero = String.duplicate("0", 40) commands = PktLine.encode("#{zero} #{sha} refs/heads/main") <> PktLine.flush()
# Feed commands first {_resp, state} = ReceivePack.feed(state, commands)
# Now feed data that exceeds the limit (256 MB) # We can't actually allocate 256MB in a test, but we can test the guard # by feeding chunks that accumulate chunk = :binary.copy("x", 1024 * 1024)
# Feed until we exceed the limit or get an error result = Enum.reduce_while(1..300, state, fn _i, st -> {resp, new_state} = ReceivePack.feed(st, chunk)
if ReceivePack.done?(new_state) do {:halt, {:done, resp}} else {:cont, new_state} end end)
case result do {:done, resp} -> # Should have error about pack too large assert resp =~ "pack_too_large"
_state -> flunk("Should have rejected oversized pack") end end end end
|