@@ -1,0 +1,347 @@
defmodule ExGitObjectstore.Pack.Reader do
@moduledoc """
Packfile reader — parses objects from .pack files.
Pack format:
- 4-byte signature: "PACK"
- 4-byte version: 2
- 4-byte object count
- N entries, each:
- variable-length header (type + uncompressed size, MSB continuation)
- for OFS_DELTA: variable-length negative offset to base object
- for REF_DELTA: 20-byte base object SHA
- zlib compressed data
- 20-byte SHA-1 checksum of all preceding content
"""
alias ExGitObjectstore.Pack.Delta
@pack_signature "PACK"
# Object types (3-bit)
@obj_commit 1
@obj_tree 2
@obj_blob 3
@obj_tag 4
@obj_ofs_delta 6
@obj_ref_delta 7
@type pack_object :: %{
type: atom(),
data: binary(),
offset: non_neg_integer()
}
@doc """
Parse all objects from a packfile binary, resolving deltas.
Returns a list of `%{type: atom, data: binary, offset: integer}`.
"""
@spec parse(binary()) :: {:ok, [pack_object()]} | {:error, term()}
def parse(<<@pack_signature, version::unsigned-big-32, count::unsigned-big-32, rest::binary>>)
when version == 2 do
case parse_entries(rest, count, [], %{}) do
{:ok, entries, _cache} -> {:ok, entries}
{:error, _} = err -> err
end
end
def parse(_), do: {:error, :invalid_pack_header}
@doc """
Read a single object at the given offset from a packfile binary.
Resolves deltas recursively using the pack data itself.
"""
@spec read_object(binary(), non_neg_integer()) :: {:ok, {atom(), binary()}} | {:error, term()}
def read_object(pack_data, offset) do
read_object(pack_data, offset, %{})
end
@doc """
Read object with a pre-populated cache of already-resolved objects.
"""
@spec read_object(binary(), non_neg_integer(), map()) ::
{:ok, {atom(), binary()}} | {:error, term()}
def read_object(pack_data, offset, cache) do
case Map.get(cache, offset) do
{type, data} ->
{:ok, {type, data}}
nil ->
do_read_object(pack_data, offset, cache)
end
end
# -- Private --
defp parse_entries(_rest, 0, acc, cache), do: {:ok, Enum.reverse(acc), cache}
defp parse_entries(rest, remaining, acc, cache) do
# We need to track the offset of each entry
# Since we're reading sequentially, we can compute it
# But pack_data isn't available here, so we'll use a different approach
# For sequential parsing, we track consumed bytes
case parse_entry_at(rest, cache, 0) do
{:ok, entry, consumed, cache} ->
<<_consumed::binary-size(consumed), new_rest::binary>> = rest
parse_entries(new_rest, remaining - 1, [entry | acc], cache)
{:error, _} = err ->
err
end
end
defp parse_entry_at(data, _cache, _base_offset) do
case parse_object_header(data) do
{:ok, type_num, _size, header_len, rest_after_header} ->
case type_num do
t when t in [@obj_commit, @obj_tree, @obj_blob, @obj_tag] ->
type = type_num_to_atom(t)
case decompress_data(rest_after_header) do
{:ok, decompressed, compressed_len} ->
entry = %{type: type, data: decompressed, offset: 0}
{:ok, entry, header_len + compressed_len, %{}}
{:error, _} = err ->
err
end
@obj_ofs_delta ->
case parse_ofs_delta_offset(rest_after_header) do
{:ok, _neg_offset, _ofs_len, rest_after_ofs} ->
case decompress_data(rest_after_ofs) do
{:ok, delta_data, compressed_len} ->
# For sequential parsing, we can't resolve OFS_DELTA without
# knowing the absolute offset. Store as unresolved delta.
entry = %{type: :ofs_delta, data: delta_data, offset: 0}
ofs_len = byte_size(rest_after_header) - byte_size(rest_after_ofs)
{:ok, entry, header_len + ofs_len + compressed_len, %{}}
{:error, _} = err ->
err
end
{:error, _} = err ->
err
end
@obj_ref_delta ->
if byte_size(rest_after_header) >= 20 do
<<_base_sha::binary-size(20), rest_after_sha::binary>> = rest_after_header
case decompress_data(rest_after_sha) do
{:ok, delta_data, compressed_len} ->
entry = %{type: :ref_delta, data: delta_data, offset: 0}
{:ok, entry, header_len + 20 + compressed_len, %{}}
{:error, _} = err ->
err
end
else
{:error, :truncated_ref_delta}
end
_ ->
{:error, {:unknown_object_type, type_num}}
end
{:error, _} = err ->
err
end
end
defp do_read_object(pack_data, offset, cache) do
<<_skip::binary-size(offset), data::binary>> = pack_data
case parse_object_header(data) do
{:ok, type_num, _size, header_len, _rest} ->
data_start = offset + header_len
case type_num do
t when t in [@obj_commit, @obj_tree, @obj_blob, @obj_tag] ->
type = type_num_to_atom(t)
<<_::binary-size(data_start), compressed::binary>> = pack_data
case decompress_data(compressed) do
{:ok, decompressed, _} -> {:ok, {type, decompressed}}
{:error, _} = err -> err
end
@obj_ofs_delta ->
<<_::binary-size(data_start), rest::binary>> = pack_data
case parse_ofs_delta_offset(rest) do
{:ok, neg_offset, ofs_len, _rest_after_ofs} ->
base_offset = offset - neg_offset
delta_start = data_start + ofs_len
<<_::binary-size(delta_start), compressed::binary>> = pack_data
with {:ok, delta_data, _} <- decompress_data(compressed),
{:ok, {base_type, base_data}} <-
read_object(pack_data, base_offset, cache),
{:ok, result} <- Delta.apply(base_data, delta_data) do
{:ok, {base_type, result}}
end
{:error, _} = err ->
err
end
@obj_ref_delta ->
<<_::binary-size(data_start), base_sha_bin::binary-size(20), compressed::binary>> =
pack_data
base_sha = Base.encode16(base_sha_bin, case: :lower)
with {:ok, delta_data, _} <- decompress_data(compressed),
{:ok, base_offset} <- find_sha_offset(pack_data, base_sha, cache),
{:ok, {base_type, base_data}} <-
read_object(pack_data, base_offset, cache),
{:ok, result} <- Delta.apply(base_data, delta_data) do
{:ok, {base_type, result}}
end
_ ->
{:error, {:unknown_object_type, type_num}}
end
{:error, _} = err ->
err
end
end
@doc """
Parse the variable-length object header.
Returns `{:ok, type_num, uncompressed_size, header_bytes_consumed, rest}`.
"""
@spec parse_object_header(binary()) ::
{:ok, non_neg_integer(), non_neg_integer(), non_neg_integer(), binary()}
| {:error, term()}
def parse_object_header(<<byte, rest::binary>>) do
type_num = Bitwise.band(Bitwise.bsr(byte, 4), 0x07)
size = Bitwise.band(byte, 0x0F)
if Bitwise.band(byte, 0x80) != 0 do
parse_size_continuation(rest, size, 4, 1, type_num)
else
{:ok, type_num, size, 1, rest}
end
end
def parse_object_header(<<>>), do: {:error, :empty_header}
defp parse_size_continuation(<<byte, rest::binary>>, size, shift, consumed, type_num) do
size = size + Bitwise.bsl(Bitwise.band(byte, 0x7F), shift)
if Bitwise.band(byte, 0x80) != 0 do
parse_size_continuation(rest, size, shift + 7, consumed + 1, type_num)
else
{:ok, type_num, size, consumed + 1, rest}
end
end
defp parse_size_continuation(<<>>, _size, _shift, _consumed, _type_num),
do: {:error, :truncated_header}
@doc """
Parse the variable-length negative offset for OFS_DELTA.
Returns `{:ok, negative_offset, bytes_consumed, rest}`.
"""
def parse_ofs_delta_offset(<<byte, rest::binary>>) do
offset = Bitwise.band(byte, 0x7F)
if Bitwise.band(byte, 0x80) != 0 do
parse_ofs_continuation(rest, offset, 1)
else
{:ok, offset, 1, rest}
end
end
def parse_ofs_delta_offset(<<>>), do: {:error, :empty_ofs_offset}
defp parse_ofs_continuation(<<byte, rest::binary>>, offset, consumed) do
offset = Bitwise.bsl(offset + 1, 7) + Bitwise.band(byte, 0x7F)
if Bitwise.band(byte, 0x80) != 0 do
parse_ofs_continuation(rest, offset, consumed + 1)
else
{:ok, offset, consumed + 1, rest}
end
end
defp parse_ofs_continuation(<<>>, _offset, _consumed), do: {:error, :truncated_ofs_offset}
defp decompress_data(compressed) do
z = :zlib.open()
try do
:zlib.inflateInit(z)
decompressed = :zlib.inflate(z, compressed)
# Get how many bytes were consumed from compressed stream
# Unfortunately :zlib doesn't tell us directly, but we can get
# the total input bytes from the decompression context
decompressed_bin = IO.iodata_to_binary(decompressed)
# Try to figure out compressed length by re-compressing
# Actually, we need the consumed bytes. Let's use a different approach.
# We'll probe by attempting to decompress increasing prefixes.
# But that's inefficient. Instead, let's use :zlib.inflateEnd to see
# if it completed cleanly, then use safeInflate for byte counting.
:zlib.inflateEnd(z)
# For the compressed length, we need to find where the zlib stream ends.
# Let's do a binary probe: decompress the whole thing and use the
# internal zlib state. Actually, the simplest approach: use :zlib.safeInflate
compressed_len = find_compressed_length(compressed)
{:ok, decompressed_bin, compressed_len}
rescue
e -> {:error, {:decompress_failed, Exception.message(e)}}
after
:zlib.close(z)
end
end
defp find_compressed_length(compressed) do
# Use a separate zlib stream with safeInflate to find consumed input bytes
z = :zlib.open()
try do
:zlib.inflateInit(z)
consume_inflate(z, compressed, 0)
after
:zlib.close(z)
end
end
defp consume_inflate(z, data, consumed) do
# Feed small chunks to safeInflate to find the boundary
chunk_size = min(byte_size(data), 4096)
<<chunk::binary-size(chunk_size), _rest::binary>> = data
case :zlib.safeInflate(z, chunk) do
{:continue, _output} ->
<<_::binary-size(chunk_size), rest::binary>> = data
consume_inflate(z, rest, consumed + chunk_size)
{:finished, _output} ->
# The stream ended somewhere in this chunk
# We consumed all of this chunk from zlib's perspective
:zlib.inflateEnd(z)
consumed + chunk_size
{:need_dictionary, _} ->
consumed + chunk_size
end
end
defp type_num_to_atom(@obj_commit), do: :commit
defp type_num_to_atom(@obj_tree), do: :tree
defp type_num_to_atom(@obj_blob), do: :blob
defp type_num_to_atom(@obj_tag), do: :tag
defp find_sha_offset(_pack_data, _sha, _cache) do
# This would need an index lookup. For now, return an error.
# The caller should have provided the offset via cache.
{:error, :ref_delta_needs_index}
end
end