ref:726fbbcf1ee0a54153160d6e485fad116e5bc8f3

Add pack protocol: pkt-line, pack writer, receive-pack, upload-pack

Phase 6 of ex_git_objectstore. Implements: - PktLine: encode/decode pkt-line format with flush/delim/sideband support - Pack.Writer: generate packfiles + .idx v2 from object lists - ReceivePack: server-side state machine for git push (ref advertisement, command parsing, pack ingestion, ref updates, status report) - UploadPack: server-side state machine for git clone/fetch (ref advertisement, want/have negotiation, pack generation with sideband) Also fixes Pack.Reader decompress_data to handle trailing data from sequential packfile parsing using binary-probed compressed length. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
SHA: 726fbbcf1ee0a54153160d6e485fad116e5bc8f3
Author: Cole Christensen <cole.christensen@macmillan.com>
Date: 2026-02-10 08:00
Parents: 3b5c899
10 files changed +1665 -37
Type
lib/ex_git_objectstore/object.ex +20 −0
@@ -119,6 +119,26 @@
defp decode_typed("tag", content), do: Tag.parse_content(content)
defp decode_typed(type, _content), do: {:error, {:unknown_type, type}}
@doc """
Encode just the content portion of a git object (no header).
"""
@spec encode_content_only(t()) :: binary()
def encode_content_only(object) do
{_type_str, content} = encode_content(object)
content
end
@doc """
Encode raw object from type atom and content binary.
Produces `"<type> <size>\\0<content>"` format.
"""
@spec encode_raw_from_type(atom(), binary()) :: binary()
def encode_raw_from_type(type, content) when is_atom(type) and is_binary(content) do
type_str = Atom.to_string(type)
header = "#{type_str} #{byte_size(content)}\0"
<<header::binary, content::binary>>
end
defp zlib_compress(data) do
z = :zlib.open()
:zlib.deflateInit(z)
lib/ex_git_objectstore/pack/reader.ex +33 −37
@@ -277,22 +277,11 @@
try do
:zlib.inflateInit(z)
decompressed = :zlib.inflate(z, compressed)
# Get how many bytes were consumed from compressed stream
# Unfortunately :zlib doesn't tell us directly, but we can get
# the total input bytes from the decompression context
decompressed_bin = IO.iodata_to_binary(decompressed)
# Try to figure out compressed length by re-compressing
# Actually, we need the consumed bytes. Let's use a different approach.
# We'll probe by attempting to decompress increasing prefixes.
# But that's inefficient. Instead, let's use :zlib.inflateEnd to see
# if it completed cleanly, then use safeInflate for byte counting.
:zlib.inflateEnd(z)
# For the compressed length, we need to find where the zlib stream ends.
# Let's do a binary probe: decompress the whole thing and use the
# internal zlib state. Actually, the simplest approach: use :zlib.safeInflate
# Find the exact compressed length by re-compressing and probing
compressed_len = find_compressed_length(compressed, byte_size(decompressed_bin))
compressed_len = find_compressed_length(compressed)
{:ok, decompressed_bin, compressed_len}
rescue
e -> {:error, {:decompress_failed, Exception.message(e)}}
@@ -301,35 +290,42 @@
end
end
# Find compressed length by binary probing: try increasingly smaller
# prefixes until decompression still produces the same output
defp find_compressed_length(compressed, expected_size) do
defp find_compressed_length(compressed) do
# Use a separate zlib stream with safeInflate to find consumed input bytes
z = :zlib.open()
total = byte_size(compressed)
probe_compressed_length(compressed, expected_size, 1, total)
end
defp probe_compressed_length(compressed, expected_size, low, high) when low < high do
mid = div(low + high, 2)
try do
:zlib.inflateInit(z)
consume_inflate(z, compressed, 0)
after
:zlib.close(z)
case try_decompress_prefix(compressed, mid) do
{:ok, size} when size == expected_size ->
# This prefix is enough — try shorter
probe_compressed_length(compressed, expected_size, low, mid)
_ ->
# This prefix is not enough — try longer
probe_compressed_length(compressed, expected_size, mid + 1, high)
end
end
defp consume_inflate(z, data, consumed) do
# Feed small chunks to safeInflate to find the boundary
chunk_size = min(byte_size(data), 4096)
<<chunk::binary-size(chunk_size), _rest::binary>> = data
defp probe_compressed_length(_compressed, _expected_size, low, _high), do: low
defp try_decompress_prefix(compressed, len) do
<<prefix::binary-size(len), _rest::binary>> = compressed
case :zlib.safeInflate(z, chunk) do
z = :zlib.open()
{:continue, _output} ->
<<_::binary-size(chunk_size), rest::binary>> = data
consume_inflate(z, rest, consumed + chunk_size)
try do
{:finished, _output} ->
# The stream ended somewhere in this chunk
# We consumed all of this chunk from zlib's perspective
:zlib.inflateEnd(z)
consumed + chunk_size
{:need_dictionary, _} ->
consumed + chunk_size
:zlib.inflateInit(z)
result = :zlib.inflate(z, prefix)
size = IO.iodata_length(result)
:zlib.inflateEnd(z)
{:ok, size}
rescue
_ -> :error
after
:zlib.close(z)
end
end
lib/ex_git_objectstore/pack/writer.ex +218 −0
@@ -1,0 +1,218 @@
defmodule ExGitObjectstore.Pack.Writer do
@moduledoc """
Generate git packfiles and their corresponding .idx files from a set of objects.
Packfile format:
- 4-byte signature: "PACK"
- 4-byte version: 2
- 4-byte object count
- N entries (type+size header + zlib compressed data)
- 20-byte SHA-1 checksum of all preceding content
Objects are stored without delta compression (full objects only).
"""
@pack_signature "PACK"
@pack_version 2
@obj_commit 1
@obj_tree 2
@obj_blob 3
@obj_tag 4
@type object_entry :: {atom(), binary(), String.t()}
@doc """
Generate a packfile from a list of `{type, raw_content, sha}` tuples.
Returns `{pack_data, pack_sha}` where pack_sha is the hex SHA-1 of the pack.
Objects should be `{:commit | :tree | :blob | :tag, raw_content, hex_sha}`.
"""
@spec generate([object_entry()]) :: {binary(), String.t()}
def generate(objects) when is_list(objects) do
count = length(objects)
header = <<@pack_signature, @pack_version::unsigned-big-32, count::unsigned-big-32>>
entries =
Enum.map(objects, fn {type, content, _sha} ->
type_num = type_to_num(type)
encode_entry(type_num, content)
end)
body = IO.iodata_to_binary([header | entries])
checksum = :crypto.hash(:sha, body)
pack_data = <<body::binary, checksum::binary>>
pack_sha = Base.encode16(checksum, case: :lower)
{pack_data, pack_sha}
end
@doc """
Generate both a packfile and its .idx v2 index.
Returns `{pack_data, idx_data, pack_sha}`.
"""
@spec generate_with_index([object_entry()]) :: {binary(), binary(), String.t()}
def generate_with_index(objects) when is_list(objects) do
count = length(objects)
header = <<@pack_signature, @pack_version::unsigned-big-32, count::unsigned-big-32>>
header_size = byte_size(header)
# Encode entries and track offsets
{entries_iodata, offset_entries} =
Enum.reduce(objects, {[], []}, fn {type, content, sha}, {entries_acc, offsets_acc} ->
offset = header_size + IO.iodata_length(entries_acc)
type_num = type_to_num(type)
entry = encode_entry(type_num, content)
crc = :erlang.crc32(IO.iodata_to_binary(entry))
{entries_acc ++ [entry], [{sha, offset, crc} | offsets_acc]}
end)
offset_entries = Enum.reverse(offset_entries)
body = IO.iodata_to_binary([header | entries_iodata])
checksum = :crypto.hash(:sha, body)
pack_data = <<body::binary, checksum::binary>>
pack_sha = Base.encode16(checksum, case: :lower)
idx_data = generate_index(offset_entries, checksum)
{pack_data, idx_data, pack_sha}
end
# Encode a single pack entry: variable-length header + zlib compressed data
defp encode_entry(type_num, content) do
size = byte_size(content)
header = encode_object_header(type_num, size)
compressed = zlib_compress(content)
[header, compressed]
end
# Encode the variable-length object header.
# First byte: bits 6-4 = type (3 bits), bits 3-0 = size (4 bits), bit 7 = continuation
# Subsequent bytes: bits 6-0 = size continuation (7 bits each), bit 7 = continuation
defp encode_object_header(type_num, size) do
# First byte: type in bits 6-4, low 4 bits of size in bits 3-0
first_byte = Bitwise.bor(Bitwise.bsl(type_num, 4), Bitwise.band(size, 0x0F))
remaining = Bitwise.bsr(size, 4)
if remaining == 0 do
<<first_byte>>
else
# Set MSB continuation bit
<<Bitwise.bor(first_byte, 0x80), encode_varint(remaining)::binary>>
end
end
defp encode_varint(0), do: <<>>
defp encode_varint(value) do
byte = Bitwise.band(value, 0x7F)
remaining = Bitwise.bsr(value, 7)
if remaining == 0 do
<<byte>>
else
<<Bitwise.bor(byte, 0x80), encode_varint(remaining)::binary>>
end
end
# Generate .idx v2 format
defp generate_index(entries, pack_checksum) do
# Sort entries by SHA
sorted = Enum.sort_by(entries, fn {sha, _offset, _crc} -> sha end)
# Build fanout table
fanout = build_fanout(sorted)
# SHA table
sha_data =
sorted
|> Enum.map(fn {sha, _offset, _crc} -> Base.decode16!(sha, case: :lower) end)
|> IO.iodata_to_binary()
# CRC table
crc_data =
sorted
|> Enum.map(fn {_sha, _offset, crc} -> <<crc::unsigned-big-32>> end)
|> IO.iodata_to_binary()
# Offset table (4-byte, with MSB flag for large offsets)
{offset_4_data, large_offsets} = build_offset_tables(sorted)
idx_body =
IO.iodata_to_binary([
# Magic + version
<<0xFF, 0x74, 0x4F, 0x63, 0, 0, 0, 2>>,
# Fanout
fanout,
# SHAs
sha_data,
# CRCs
crc_data,
# 4-byte offsets
offset_4_data,
# Large offsets (if any)
large_offsets,
# Pack checksum
pack_checksum
])
# Index checksum
idx_checksum = :crypto.hash(:sha, idx_body)
<<idx_body::binary, idx_checksum::binary>>
end
defp build_fanout(sorted_entries) do
# 256 entries, each is cumulative count of objects with first SHA byte <= N
sha_first_bytes =
Enum.map(sorted_entries, fn {sha, _offset, _crc} ->
<<first_byte, _rest::binary>> = Base.decode16!(sha, case: :lower)
first_byte
end)
fanout =
for i <- 0..255 do
Enum.count(sha_first_bytes, fn b -> b <= i end)
end
fanout
|> Enum.map(fn count -> <<count::unsigned-big-32>> end)
|> IO.iodata_to_binary()
end
defp build_offset_tables(sorted_entries) do
{offset_4_entries, large_offset_entries} =
sorted_entries
|> Enum.with_index()
|> Enum.reduce({[], []}, fn {{_sha, offset, _crc}, _idx}, {small, large} ->
if offset > 0x7FFFFFFF do
large_idx = length(large)
flagged = Bitwise.bor(large_idx, 0x80000000)
{[<<flagged::unsigned-big-32>> | small], [<<offset::unsigned-big-64>> | large]}
else
{[<<offset::unsigned-big-32>> | small], large}
end
end)
offset_4 = offset_4_entries |> Enum.reverse() |> IO.iodata_to_binary()
large = large_offset_entries |> Enum.reverse() |> IO.iodata_to_binary()
{offset_4, large}
end
defp type_to_num(:commit), do: @obj_commit
defp type_to_num(:tree), do: @obj_tree
defp type_to_num(:blob), do: @obj_blob
defp type_to_num(:tag), do: @obj_tag
defp zlib_compress(data) do
z = :zlib.open()
:zlib.deflateInit(z)
compressed = :zlib.deflate(z, data, :finish)
:zlib.deflateEnd(z)
:zlib.close(z)
IO.iodata_to_binary(compressed)
end
end
lib/ex_git_objectstore/protocol/pkt_line.ex +167 −0
@@ -1,0 +1,167 @@
defmodule ExGitObjectstore.Protocol.PktLine do
@moduledoc """
Git pkt-line format encoding and decoding.
Pkt-line format uses a 4-byte hex length prefix:
- `0000` — flush packet (end of message)
- `0001` — delimiter packet (separates sections in v2)
- `0002` — response-end packet
- `NNNN<data>` — data packet where NNNN is total length (including the 4 prefix bytes)
Minimum data packet length is `0005` (4 prefix + 1 data byte).
Maximum is `ffff` (65535 bytes total, 65531 data bytes).
"""
@flush "0000"
@delim "0001"
@response_end "0002"
@max_data_len 65520
@doc """
Encode a single data line as a pkt-line.
Appends a newline unless the data already ends with one.
"""
@spec encode(String.t()) :: binary()
def encode(data) when is_binary(data) do
data = if String.ends_with?(data, "\n"), do: data, else: data <> "\n"
len = byte_size(data) + 4
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0")
<<hex_len::binary, data::binary>>
end
@doc """
Encode raw binary data as a pkt-line (no newline appended).
"""
@spec encode_raw(binary()) :: binary()
def encode_raw(data) when is_binary(data) do
len = byte_size(data) + 4
hex_len = len |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(4, "0")
<<hex_len::binary, data::binary>>
end
@doc """
Return a flush packet.
"""
@spec flush() :: binary()
def flush, do: @flush
@doc """
Return a delimiter packet.
"""
@spec delim() :: binary()
def delim, do: @delim
@doc """
Return a response-end packet.
"""
@spec response_end() :: binary()
def response_end, do: @response_end
@doc """
Encode a list of lines, followed by a flush.
"""
@spec encode_lines([String.t()]) :: binary()
def encode_lines(lines) do
encoded = Enum.map(lines, &encode/1)
IO.iodata_to_binary(encoded ++ [@flush])
end
@doc """
Decode pkt-line data into a list of packets.
Returns `{:ok, packets, rest}` where each packet is:
- `:flush` — flush packet
- `:delim` — delimiter packet
- `:response_end` — response-end packet
- `{:data, binary()}` — data packet (with trailing newline stripped)
"""
@spec decode(binary()) :: {:ok, [packet()], binary()} | {:error, term()}
@type packet :: :flush | :delim | :response_end | {:data, binary()}
def decode(data), do: decode_loop(data, [])
@doc """
Decode a single pkt-line from the front of the binary.
Returns `{:ok, packet, rest}` or `{:need_more, data}`.
"""
@spec decode_one(binary()) ::
{:ok, packet(), binary()} | {:need_more, binary()} | {:error, term()}
def decode_one(<<@flush, rest::binary>>), do: {:ok, :flush, rest}
def decode_one(<<@delim, rest::binary>>), do: {:ok, :delim, rest}
def decode_one(<<@response_end, rest::binary>>), do: {:ok, :response_end, rest}
def decode_one(<<hex_len::binary-size(4), _rest::binary>> = data) do
case Integer.parse(hex_len, 16) do
{len, ""} when len >= 4 ->
if byte_size(data) >= len do
<<_hex::binary-size(4), payload::binary-size(len - 4), rest::binary>> = data
# Strip trailing newline if present
payload = String.trim_trailing(payload, "\n")
{:ok, {:data, payload}, rest}
else
{:need_more, data}
end
{_, ""} ->
{:error, {:invalid_pkt_len, hex_len}}
_ ->
{:error, {:invalid_pkt_hex, hex_len}}
end
end
def decode_one(data) when byte_size(data) < 4, do: {:need_more, data}
def decode_one(_), do: {:error, :invalid_pkt_line}
@doc """
Split sideband data (band 1 = pack data, band 2 = progress, band 3 = error).
"""
@spec decode_sideband(binary()) ::
{:pack, binary()} | {:progress, binary()} | {:error_msg, binary()} | {:error, term()}
def decode_sideband(<<1, rest::binary>>), do: {:pack, rest}
def decode_sideband(<<2, rest::binary>>), do: {:progress, rest}
def decode_sideband(<<3, rest::binary>>), do: {:error_msg, rest}
def decode_sideband(_), do: {:error, :invalid_sideband}
@doc """
Encode sideband data for band 1 (pack data).
Splits into max-size chunks if needed.
"""
@spec encode_sideband(1 | 2 | 3, binary()) :: iodata()
def encode_sideband(band, data) when band in [1, 2, 3] do
chunks = chunk_binary(data, @max_data_len)
Enum.map(chunks, fn chunk ->
encode_raw(<<band, chunk::binary>>)
end)
end
# -- Private --
defp decode_loop(<<>>, acc), do: {:ok, Enum.reverse(acc), <<>>}
defp decode_loop(data, acc) do
case decode_one(data) do
{:ok, :flush, rest} ->
{:ok, Enum.reverse([:flush | acc]), rest}
{:ok, packet, rest} ->
decode_loop(rest, [packet | acc])
{:need_more, rest} ->
{:ok, Enum.reverse(acc), rest}
{:error, _} = err ->
err
end
end
defp chunk_binary(<<>>, _size), do: []
defp chunk_binary(data, size) when byte_size(data) <= size, do: [data]
defp chunk_binary(data, size) do
<<chunk::binary-size(size), rest::binary>> = data
[chunk | chunk_binary(rest, size)]
end
end
lib/ex_git_objectstore/protocol/receive_pack.ex +323 −0
@@ -1,0 +1,323 @@
defmodule ExGitObjectstore.Protocol.ReceivePack do
@moduledoc """
State machine for the git receive-pack protocol (server side of `git push`).
Flow:
1. Server advertises refs (capabilities + current refs)
2. Client sends ref update commands
3. Client sends a packfile containing the objects
4. Server processes the pack and updates refs
5. Server sends a report-status
This is a pure functional state machine — no processes. The caller feeds data
in and reads response data out.
"""
alias ExGitObjectstore.{Object, Ref, Repo}
alias ExGitObjectstore.Pack.Reader
alias ExGitObjectstore.Protocol.PktLine
@zero_sha String.duplicate("0", 40)
@capabilities "report-status delete-refs ofs-delta side-band-64k"
@type command :: %{
ref: String.t(),
old_sha: String.t(),
new_sha: String.t()
}
@type state :: %__MODULE__{
repo: Repo.t(),
phase: :advertise | :commands | :pack | :done,
commands: [command()],
pack_buffer: binary(),
result: :ok | {:error, term()}
}
defstruct [
:repo,
phase: :advertise,
commands: [],
pack_buffer: <<>>,
result: nil
]
@doc """
Create a new receive-pack state machine and generate the ref advertisement.
Returns `{advertisement_data, state}`.
"""
@spec init(Repo.t()) :: {binary(), state()}
def init(%Repo{} = repo) do
state = %__MODULE__{repo: repo, phase: :commands}
advert = build_advertisement(repo)
{advert, state}
end
@doc """
Feed data from the client into the state machine.
Returns `{response_data, new_state}`.
In the `:commands` phase, parses ref update commands.
In the `:pack` phase, buffers pack data until complete.
"""
@spec feed(state(), binary()) :: {binary(), state()}
def feed(%__MODULE__{phase: :commands} = state, data) do
case parse_commands(data) do
{:ok, commands, rest} ->
state = %{state | commands: commands, phase: :pack, pack_buffer: rest}
# Check if the pack is already complete in the buffer
maybe_process_pack(state)
{:need_more, _} ->
# Shouldn't happen in practice — commands come in one batch
{<<>>, state}
{:error, reason} ->
report = build_error_report(reason)
{report, %{state | phase: :done, result: {:error, reason}}}
end
end
def feed(%__MODULE__{phase: :pack} = state, data) do
state = %{state | pack_buffer: state.pack_buffer <> data}
maybe_process_pack(state)
end
def feed(%__MODULE__{phase: :done} = state, _data) do
{<<>>, state}
end
@doc """
Check if the protocol exchange is complete.
"""
@spec done?(state()) :: boolean()
def done?(%__MODULE__{phase: :done}), do: true
def done?(_), do: false
# -- Private --
defp build_advertisement(repo) do
refs = list_all_refs(repo)
case refs do
[] ->
# Empty repo — advertise capabilities with zero SHA
line = "#{@zero_sha} capabilities^{}\0 #{@capabilities}"
PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] ->
first_line = "#{first_sha} #{first_ref}\0 #{@capabilities}"
lines =
[PktLine.encode(first_line)] ++
Enum.map(rest, fn {ref, sha} ->
PktLine.encode("#{sha} #{ref}")
end) ++
[PktLine.flush()]
IO.iodata_to_binary(lines)
end
end
defp list_all_refs(repo) do
heads =
case Ref.list(repo, "refs/heads/") do
{:ok, refs} -> refs
_ -> []
end
tags =
case Ref.list(repo, "refs/tags/") do
{:ok, refs} -> refs
_ -> []
end
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
end
defp parse_commands(data) do
parse_command_lines(data, [])
end
defp parse_command_lines(data, acc) do
case PktLine.decode_one(data) do
{:ok, :flush, rest} ->
if acc == [] do
# No commands — client is just probing
{:ok, [], rest}
else
{:ok, Enum.reverse(acc), rest}
end
{:ok, {:data, line}, rest} ->
case parse_command_line(line) do
{:ok, cmd} ->
parse_command_lines(rest, [cmd | acc])
{:error, _} = err ->
err
end
{:need_more, _} ->
{:need_more, data}
{:error, _} = err ->
err
end
end
defp parse_command_line(line) do
# Format: "<old-sha> <new-sha> <ref-name>[\0<capabilities>]"
# Strip capabilities if present
line =
case String.split(line, "\0", parts: 2) do
[cmd, _caps] -> cmd
[cmd] -> cmd
end
case String.split(line, " ", parts: 3) do
[old_sha, new_sha, ref] when byte_size(old_sha) == 40 and byte_size(new_sha) == 40 ->
{:ok, %{ref: ref, old_sha: old_sha, new_sha: new_sha}}
_ ->
{:error, {:invalid_command, line}}
end
end
defp maybe_process_pack(%{commands: commands} = state) do
# If all commands are deletes (new_sha == 0000...), no pack is sent
all_deletes? = Enum.all?(commands, fn cmd -> cmd.new_sha == @zero_sha end)
if all_deletes? or commands == [] do
process_ref_updates(state)
else
# Check if we have a complete packfile
case check_pack_complete(state.pack_buffer) do
:complete ->
process_pack_and_refs(state)
:incomplete ->
{<<>>, state}
{:error, reason} ->
report = build_error_report(reason)
{report, %{state | phase: :done, result: {:error, reason}}}
end
end
end
defp check_pack_complete(
<<"PACK", _version::32, _count::unsigned-big-32, _rest::binary>> = data
) do
# A complete pack has: header (12 bytes) + entries + 20-byte checksum
# We can't easily know the total size without parsing all entries,
# so we verify the trailing SHA-1 checksum
if byte_size(data) > 32 do
body_len = byte_size(data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = data
expected = :crypto.hash(:sha, body)
if checksum == expected do
:complete
else
# May be incomplete — more data coming
:incomplete
end
else
:incomplete
end
end
defp check_pack_complete(<<>>), do: :incomplete
defp check_pack_complete(_), do: :incomplete
defp process_pack_and_refs(state) do
case store_pack_objects(state.repo, state.pack_buffer) do
:ok ->
process_ref_updates(state)
{:error, reason} ->
report = build_error_report(reason)
{report, %{state | phase: :done, result: {:error, reason}}}
end
end
defp store_pack_objects(repo, pack_data) do
case Reader.parse(pack_data) do
{:ok, entries} ->
Enum.reduce_while(entries, :ok, fn entry, :ok ->
raw = Object.encode_raw_from_type(entry.type, entry.data)
sha = :crypto.hash(:sha, raw) |> Base.encode16(case: :lower)
compressed = zlib_compress(raw)
case Repo.storage_call(repo, :put_object, [sha, compressed]) do
:ok -> {:cont, :ok}
{:error, _} = err -> {:halt, err}
end
end)
{:error, _} = err ->
err
end
end
defp process_ref_updates(state) do
results =
Enum.map(state.commands, fn cmd ->
result = apply_ref_command(state.repo, cmd)
{cmd.ref, result}
end)
report = build_report(results)
{report, %{state | phase: :done, result: :ok}}
end
defp apply_ref_command(repo, %{old_sha: @zero_sha, new_sha: new_sha, ref: ref}) do
# Create ref
Ref.put(repo, ref, new_sha, nil)
end
defp apply_ref_command(repo, %{new_sha: @zero_sha, ref: ref}) do
# Delete ref
Ref.delete(repo, ref)
end
defp apply_ref_command(repo, %{old_sha: old_sha, new_sha: new_sha, ref: ref}) do
# Update ref (CAS)
Ref.put(repo, ref, new_sha, old_sha)
end
defp build_report(results) do
lines =
[PktLine.encode("unpack ok")] ++
Enum.map(results, fn
{ref, :ok} ->
PktLine.encode("ok #{ref}")
{ref, {:error, reason}} ->
PktLine.encode("ng #{ref} #{inspect(reason)}")
end) ++
[PktLine.flush()]
IO.iodata_to_binary(lines)
end
defp build_error_report(reason) do
lines = [
PktLine.encode("unpack #{inspect(reason)}"),
PktLine.flush()
]
IO.iodata_to_binary(lines)
end
defp zlib_compress(data) do
z = :zlib.open()
:zlib.deflateInit(z)
compressed = :zlib.deflate(z, data, :finish)
:zlib.deflateEnd(z)
:zlib.close(z)
IO.iodata_to_binary(compressed)
end
end
lib/ex_git_objectstore/protocol/upload_pack.ex +258 −0
@@ -1,0 +1,258 @@
defmodule ExGitObjectstore.Protocol.UploadPack do
@moduledoc """
State machine for the git upload-pack protocol (server side of `git fetch/clone`).
Flow:
1. Server advertises refs (capabilities + current refs)
2. Client sends want/have lines (what they want and what they already have)
3. Server computes the minimal set of objects and sends a packfile
This is a pure functional state machine — no processes.
"""
alias ExGitObjectstore.{Object, ObjectResolver, Ref, Repo}
alias ExGitObjectstore.Object.{Blob, Commit, Tree}
alias ExGitObjectstore.Pack.Writer
alias ExGitObjectstore.Protocol.PktLine
@zero_sha String.duplicate("0", 40)
@capabilities "multi_ack_detailed side-band-64k ofs-delta"
@type state :: %__MODULE__{
repo: Repo.t(),
phase: :advertise | :negotiation | :pack | :done,
wants: [String.t()],
haves: [String.t()],
common: [String.t()]
}
defstruct [
:repo,
phase: :advertise,
wants: [],
haves: [],
common: []
]
@doc """
Create a new upload-pack state machine and generate the ref advertisement.
Returns `{advertisement_data, state}`.
"""
@spec init(Repo.t()) :: {binary(), state()}
def init(%Repo{} = repo) do
state = %__MODULE__{repo: repo, phase: :negotiation}
advert = build_advertisement(repo)
{advert, state}
end
@doc """
Feed data from the client into the state machine.
Returns `{response_data, new_state}`.
"""
@spec feed(state(), binary()) :: {binary(), state()}
def feed(%__MODULE__{phase: :negotiation} = state, data) do
case parse_wants_haves(data) do
{:ok, wants, haves, :done} ->
state = %{state | wants: wants, haves: haves, phase: :pack}
generate_pack_response(state)
{:ok, wants, haves, :continue} ->
# Client wants more negotiation rounds — simplified: just ACK and continue
state = %{state | wants: wants ++ state.wants, haves: haves ++ state.haves}
nak = PktLine.encode("NAK")
{nak, state}
{:error, _reason} ->
nak = PktLine.encode("NAK")
{nak, %{state | phase: :done}}
end
end
def feed(%__MODULE__{phase: :done} = state, _data) do
{<<>>, state}
end
@doc """
Check if the protocol exchange is complete.
"""
@spec done?(state()) :: boolean()
def done?(%__MODULE__{phase: :done}), do: true
def done?(_), do: false
# -- Private --
defp build_advertisement(repo) do
refs = list_all_refs(repo)
case refs do
[] ->
line = "#{@zero_sha} capabilities^{}\0 #{@capabilities}"
PktLine.encode(line) <> PktLine.flush()
[{first_ref, first_sha} | rest] ->
first_line = "#{first_sha} #{first_ref}\0 #{@capabilities}"
lines =
[PktLine.encode(first_line)] ++
Enum.map(rest, fn {ref, sha} ->
PktLine.encode("#{sha} #{ref}")
end) ++
[PktLine.flush()]
IO.iodata_to_binary(lines)
end
end
defp list_all_refs(repo) do
heads =
case Ref.list(repo, "refs/heads/") do
{:ok, refs} -> refs
_ -> []
end
tags =
case Ref.list(repo, "refs/tags/") do
{:ok, refs} -> refs
_ -> []
end
(heads ++ tags)
|> Enum.sort_by(fn {ref, _sha} -> ref end)
end
defp parse_wants_haves(data) do
parse_wh_lines(data, [], [], nil)
end
defp parse_wh_lines(data, wants, haves, _terminator) do
case PktLine.decode_one(data) do
{:ok, :flush, _rest} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :done}
{:ok, {:data, "done"}, _rest} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :done}
{:ok, {:data, "want " <> sha_and_caps}, rest} ->
# First want line may have capabilities after SHA
sha = sha_and_caps |> String.split(" ", parts: 2) |> List.first() |> String.trim()
parse_wh_lines(rest, [sha | wants], haves, nil)
{:ok, {:data, "have " <> sha}, rest} ->
sha = String.trim(sha)
parse_wh_lines(rest, wants, [sha | haves], nil)
{:ok, {:data, _other}, rest} ->
# Skip unknown lines
parse_wh_lines(rest, wants, haves, nil)
{:need_more, _} ->
{:ok, Enum.reverse(wants), Enum.reverse(haves), :continue}
{:error, _} = err ->
err
end
end
defp generate_pack_response(state) do
# Collect all objects needed for the wants, excluding objects reachable from haves
case collect_objects(state.repo, state.wants, state.haves) do
{:ok, objects} ->
# Generate packfile
{pack_data, _pack_sha} = Writer.generate(objects)
# Build response: NAK + sideband pack data
nak = PktLine.encode("NAK")
sideband_data =
PktLine.encode_sideband(1, pack_data)
|> IO.iodata_to_binary()
# Flush to end
response = IO.iodata_to_binary([nak, sideband_data, PktLine.flush()])
{response, %{state | phase: :done}}
{:error, reason} ->
nak = PktLine.encode("ERR #{inspect(reason)}")
{nak, %{state | phase: :done}}
end
end
defp collect_objects(repo, wants, haves) do
# Walk from each want SHA and collect all reachable objects
have_set = MapSet.new(haves)
try do
objects =
wants
|> Enum.flat_map(fn sha ->
collect_reachable(repo, sha, have_set)
end)
|> Enum.uniq_by(fn {_type, _data, sha} -> sha end)
{:ok, objects}
rescue
e -> {:error, Exception.message(e)}
end
end
defp collect_reachable(repo, sha, exclude_set) do
if MapSet.member?(exclude_set, sha) do
[]
else
case ObjectResolver.read(repo, sha) do
{:ok, %Commit{} = commit} ->
# Collect the tree and all referenced objects
tree_objects = collect_tree_objects(repo, commit.tree)
# Recurse into parents (but stop at haves)
parent_objects =
Enum.flat_map(commit.parents, fn parent_sha ->
collect_reachable(repo, parent_sha, exclude_set)
end)
[{:commit, Object.encode_content_only(commit), sha} | tree_objects ++ parent_objects]
{:ok, %Tree{} = tree} ->
collect_tree_entry_objects(repo, tree, sha)
{:ok, %Blob{content: content}} ->
[{:blob, content, sha}]
{:error, _} ->
[]
end
end
end
defp collect_tree_objects(repo, tree_sha) do
case ObjectResolver.read(repo, tree_sha) do
{:ok, %Tree{} = tree} ->
collect_tree_entry_objects(repo, tree, tree_sha)
_ ->
[]
end
end
defp collect_tree_entry_objects(repo, %Tree{} = tree, tree_sha) do
tree_content = Tree.encode_content(tree)
entry = [{:tree, tree_content, tree_sha}]
child_objects =
Enum.flat_map(tree.entries, fn
%{mode: "40000", sha: sha} ->
collect_tree_objects(repo, sha)
%{sha: sha} ->
case ObjectResolver.read(repo, sha) do
{:ok, %Blob{content: content}} ->
[{:blob, content, sha}]
_ ->
[]
end
end)
entry ++ child_objects
end
end
test/ex_git_objectstore/pack/writer_test.exs +172 −0
@@ -1,0 +1,172 @@
defmodule ExGitObjectstore.Pack.WriterTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.Pack.{Writer, Reader, Index}
alias ExGitObjectstore.Object
alias ExGitObjectstore.Object.{Blob, Tree, Commit}
describe "generate" do
test "generates valid packfile with a single blob" do
content = "hello world\n"
blob = Blob.from_content(content)
sha = Object.hash(blob)
{pack_data, pack_sha} = Writer.generate([{:blob, content, sha}])
# Verify header
assert <<"PACK", 2::unsigned-big-32, 1::unsigned-big-32, _rest::binary>> = pack_data
# Verify checksum
body_len = byte_size(pack_data) - 20
<<body::binary-size(body_len), checksum::binary-size(20)>> = pack_data
assert checksum == :crypto.hash(:sha, body)
# pack_sha should be hex of checksum
assert pack_sha == Base.encode16(checksum, case: :lower)
end
test "generated pack can be parsed by Reader" do
content = "test content\n"
blob = Blob.from_content(content)
sha = Object.hash(blob)
{pack_data, _pack_sha} = Writer.generate([{:blob, content, sha}])
assert {:ok, entries} = Reader.parse(pack_data)
assert length(entries) == 1
assert hd(entries).type == :blob
assert hd(entries).data == content
end
test "multiple objects" do
blob1 = Blob.from_content("one\n")
sha1 = Object.hash(blob1)
blob2 = Blob.from_content("two\n")
sha2 = Object.hash(blob2)
objects = [
{:blob, "one\n", sha1},
{:blob, "two\n", sha2}
]
{pack_data, _} = Writer.generate(objects)
assert {:ok, entries} = Reader.parse(pack_data)
assert length(entries) == 2
contents = Enum.map(entries, & &1.data) |> Enum.sort()
assert contents == ["one\n", "two\n"]
end
test "commit object in pack" do
commit = %Commit{
tree: String.duplicate("a", 40),
parents: [],
author: "Test <test@test.com> 1000000000 +0000",
committer: "Test <test@test.com> 1000000000 +0000",
message: "initial\n"
}
commit_content = Commit.encode_content(commit)
sha = Object.hash(commit)
{pack_data, _} = Writer.generate([{:commit, commit_content, sha}])
assert {:ok, entries} = Reader.parse(pack_data)
assert length(entries) == 1
assert hd(entries).type == :commit
end
test "tree object in pack" do
blob = Blob.from_content("content\n")
blob_sha = Object.hash(blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
tree_content = Tree.encode_content(tree)
tree_sha = Object.hash(tree)
{pack_data, _} = Writer.generate([{:tree, tree_content, tree_sha}])
assert {:ok, entries} = Reader.parse(pack_data)
assert length(entries) == 1
assert hd(entries).type == :tree
end
end
describe "generate_with_index" do
test "generates both pack and valid index" do
blob = Blob.from_content("hello\n")
sha = Object.hash(blob)
{pack_data, idx_data, _pack_sha} =
Writer.generate_with_index([{:blob, "hello\n", sha}])
# Verify pack
assert <<"PACK", _::binary>> = pack_data
# Verify index
assert {:ok, index} = Index.parse(idx_data)
assert index.count == 1
assert Index.member?(index, sha)
end
test "index offsets point to correct objects" do
blob1 = Blob.from_content("one\n")
sha1 = Object.hash(blob1)
blob2 = Blob.from_content("two\n")
sha2 = Object.hash(blob2)
objects = [
{:blob, "one\n", sha1},
{:blob, "two\n", sha2}
]
{pack_data, idx_data, _} = Writer.generate_with_index(objects)
assert {:ok, index} = Index.parse(idx_data)
assert index.count == 2
# Verify each object can be read at its index offset
for sha <- [sha1, sha2] do
assert {:ok, offset} = Index.lookup(index, sha)
assert {:ok, {:blob, _content}} = Reader.read_object(pack_data, offset)
end
end
test "multiple object types" do
blob = Blob.from_content("content\n")
blob_sha = Object.hash(blob)
blob_content = "content\n"
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
tree_sha = Object.hash(tree)
tree_content = Tree.encode_content(tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <test@test.com> 1000000000 +0000",
committer: "Test <test@test.com> 1000000000 +0000",
message: "test\n"
}
commit_sha = Object.hash(commit)
commit_content = Commit.encode_content(commit)
objects = [
{:blob, blob_content, blob_sha},
{:tree, tree_content, tree_sha},
{:commit, commit_content, commit_sha}
]
{_pack_data, idx_data, _} = Writer.generate_with_index(objects)
assert {:ok, index} = Index.parse(idx_data)
assert index.count == 3
# Verify all objects are findable
assert Index.member?(index, blob_sha)
assert Index.member?(index, tree_sha)
assert Index.member?(index, commit_sha)
end
end
end
test/ex_git_objectstore/protocol/pkt_line_test.exs +149 −0
@@ -1,0 +1,149 @@
defmodule ExGitObjectstore.Protocol.PktLineTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.Protocol.PktLine
describe "encode" do
test "encodes data with length prefix and newline" do
result = PktLine.encode("hello")
# "hello\n" = 6 bytes + 4 prefix = 10 = 000a
assert result == "000ahello\n"
end
test "does not double-add newline" do
result = PktLine.encode("hello\n")
assert result == "000ahello\n"
end
test "encodes empty-ish line" do
result = PktLine.encode("x")
# "x\n" = 2 bytes + 4 prefix = 6 = 0006
assert result == "0006x\n"
end
end
describe "encode_raw" do
test "encodes without appending newline" do
result = PktLine.encode_raw("hello")
# "hello" = 5 bytes + 4 prefix = 9 = 0009
assert result == "0009hello"
end
test "preserves binary data" do
data = <<1, 2, 3>>
result = PktLine.encode_raw(data)
assert result == "0007" <> data
end
end
describe "flush/delim/response_end" do
test "flush packet" do
assert PktLine.flush() == "0000"
end
test "delim packet" do
assert PktLine.delim() == "0001"
end
test "response end packet" do
assert PktLine.response_end() == "0002"
end
end
describe "encode_lines" do
test "encodes lines with flush at end" do
result = PktLine.encode_lines(["hello", "world"])
assert result == "000ahello\n000aworld\n0000"
end
test "empty list produces just flush" do
result = PktLine.encode_lines([])
assert result == "0000"
end
end
describe "decode_one" do
test "decodes flush" do
assert {:ok, :flush, "rest"} = PktLine.decode_one("0000rest")
end
test "decodes delim" do
assert {:ok, :delim, ""} = PktLine.decode_one("0001")
end
test "decodes data line" do
assert {:ok, {:data, "hello"}, ""} = PktLine.decode_one("000ahello\n")
end
test "decodes data without newline" do
assert {:ok, {:data, "hello"}, ""} = PktLine.decode_one("0009hello")
end
test "returns need_more for incomplete data" do
assert {:need_more, "000a"} = PktLine.decode_one("000a")
end
test "returns need_more for too-short input" do
assert {:need_more, "00"} = PktLine.decode_one("00")
end
end
describe "decode" do
test "decodes multiple lines" do
data = "000ahello\n000aworld\n0000"
assert {:ok, [{:data, "hello"}, {:data, "world"}, :flush], ""} = PktLine.decode(data)
end
test "handles incomplete data" do
data = "000ahello\n000a"
assert {:ok, [{:data, "hello"}], "000a"} = PktLine.decode(data)
end
test "decodes empty input" do
assert {:ok, [], ""} = PktLine.decode("")
end
end
describe "roundtrip" do
test "encode then decode" do
original = "some data here"
encoded = PktLine.encode(original)
assert {:ok, {:data, ^original}, ""} = PktLine.decode_one(encoded)
end
test "encode_lines then decode" do
lines = ["line1", "line2", "line3"]
encoded = PktLine.encode_lines(lines)
{:ok, decoded, ""} = PktLine.decode(encoded)
data_lines =
Enum.filter(decoded, fn
{:data, _} -> true
_ -> false
end)
|> Enum.map(fn {:data, d} -> d end)
assert data_lines == lines
end
end
describe "sideband" do
test "decode sideband band 1 (pack data)" do
assert {:pack, "data"} = PktLine.decode_sideband(<<1, "data">>)
end
test "decode sideband band 2 (progress)" do
assert {:progress, "msg"} = PktLine.decode_sideband(<<2, "msg">>)
end
test "decode sideband band 3 (error)" do
assert {:error_msg, "oops"} = PktLine.decode_sideband(<<3, "oops">>)
end
test "encode sideband" do
result = PktLine.encode_sideband(1, "hello") |> IO.iodata_to_binary()
# Sideband: band byte (1) + "hello" = 6 bytes + 4 prefix = 10
assert result == "000a" <> <<1, "hello">>
end
end
end
test/ex_git_objectstore/protocol/receive_pack_test.exs +160 −0
@@ -1,0 +1,160 @@
defmodule ExGitObjectstore.Protocol.ReceivePackTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.Protocol.{ReceivePack, PktLine}
alias ExGitObjectstore.{Object, Ref}
alias ExGitObjectstore.Object.{Blob, Commit, Tree}
alias ExGitObjectstore.Pack.Writer
alias ExGitObjectstore.Test.RepoHelper
describe "init" do
test "advertises refs for empty repo" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, state} = ReceivePack.init(repo)
assert state.phase == :commands
# Should contain capabilities and flush
{:ok, packets, ""} = PktLine.decode(advert)
assert List.last(packets) == :flush
# First line should have capabilities
{:data, first_line} = hd(packets)
assert String.contains?(first_line, "report-status")
end
test "advertises existing refs" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
# Create a commit so we have a ref to advertise
blob = Blob.from_content("hello\n")
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: "init\n"
}
{:ok, commit_sha} = Object.write(repo, commit)
:ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = ReceivePack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
# Should have at least one ref line + flush
data_lines =
Enum.filter(packets, fn
{:data, _} -> true
_ -> false
end)
assert length(data_lines) >= 1
{:data, first_line} = hd(data_lines)
assert String.contains?(first_line, commit_sha)
assert String.contains?(first_line, "refs/heads/main")
end
end
describe "feed - ref updates" do
test "create a new branch via push" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
# Create objects
blob = Blob.from_content("hello\n")
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: "init\n"
}
{:ok, commit_sha} = Object.write(repo, commit)
{_advert, state} = ReceivePack.init(repo)
# Build client commands: create refs/heads/main
zero = String.duplicate("0", 40)
commands = PktLine.encode("#{zero} #{commit_sha} refs/heads/main") <> PktLine.flush()
# Build a pack containing the objects
objects = [
{:blob, "hello\n", blob_sha},
{:tree, Tree.encode_content(tree), tree_sha},
{:commit, Commit.encode_content(commit), commit_sha}
]
{pack_data, _} = Writer.generate(objects)
# Feed commands + pack
{response, state} = ReceivePack.feed(state, commands <> pack_data)
assert ReceivePack.done?(state)
# Parse the report
{:ok, packets, _} = PktLine.decode(response)
data_lines = for {:data, d} <- packets, do: d
assert "unpack ok" in data_lines
assert "ok refs/heads/main" in data_lines
# Verify the ref was created
assert {:ok, ^commit_sha} = Ref.get(repo, "refs/heads/main")
end
test "delete a branch" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
# Create a commit and ref
blob = Blob.from_content("hello\n")
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: [],
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: "init\n"
}
{:ok, commit_sha} = Object.write(repo, commit)
:ok = Ref.put(repo, "refs/heads/feature", commit_sha, nil)
{_advert, state} = ReceivePack.init(repo)
# Delete command: old_sha → zero
zero = String.duplicate("0", 40)
commands = PktLine.encode("#{commit_sha} #{zero} refs/heads/feature") <> PktLine.flush()
# No pack needed for deletes
{response, state} = ReceivePack.feed(state, commands)
assert ReceivePack.done?(state)
{:ok, packets, _} = PktLine.decode(response)
data_lines = for {:data, d} <- packets, do: d
assert "unpack ok" in data_lines
assert "ok refs/heads/feature" in data_lines
# Verify ref was deleted
assert {:error, _} = Ref.get(repo, "refs/heads/feature")
end
end
end
test/ex_git_objectstore/protocol/upload_pack_test.exs +165 −0
@@ -1,0 +1,165 @@
defmodule ExGitObjectstore.Protocol.UploadPackTest do
use ExUnit.Case, async: true
alias ExGitObjectstore.Protocol.{UploadPack, PktLine}
alias ExGitObjectstore.{Object, Ref}
alias ExGitObjectstore.Object.{Blob, Commit, Tree}
alias ExGitObjectstore.Test.RepoHelper
defp create_commit(repo, content, message, parents \\ []) do
blob = Blob.from_content(content)
{:ok, blob_sha} = Object.write(repo, blob)
tree = Tree.new([%{mode: "100644", name: "file.txt", sha: blob_sha}])
{:ok, tree_sha} = Object.write(repo, tree)
commit = %Commit{
tree: tree_sha,
parents: parents,
author: "Test <t@t.com> 1000000000 +0000",
committer: "Test <t@t.com> 1000000000 +0000",
message: message
}
{:ok, commit_sha} = Object.write(repo, commit)
{commit_sha, blob_sha, tree_sha}
end
describe "init" do
test "advertises refs for empty repo" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{advert, state} = UploadPack.init(repo)
assert state.phase == :negotiation
{:ok, packets, ""} = PktLine.decode(advert)
assert List.last(packets) == :flush
end
test "advertises refs with commit" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{commit_sha, _, _} = create_commit(repo, "hello\n", "init\n")
:ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{advert, _state} = UploadPack.init(repo)
{:ok, packets, ""} = PktLine.decode(advert)
data_lines = for {:data, d} <- packets, do: d
first = hd(data_lines)
assert String.contains?(first, commit_sha)
assert String.contains?(first, "refs/heads/main")
end
end
describe "feed - clone (wants only, no haves)" do
test "sends packfile with all objects for a clone" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
{commit_sha, _blob_sha, _tree_sha} = create_commit(repo, "hello\n", "init\n")
:ok = Ref.put(repo, "refs/heads/main", commit_sha, nil)
{_advert, state} = UploadPack.init(repo)
# Client sends want + done
client_data =
PktLine.encode("want #{commit_sha}") <>
PktLine.flush() <>
PktLine.encode("done")
{response, state} = UploadPack.feed(state, client_data)
assert UploadPack.done?(state)
assert byte_size(response) > 0
# The response should contain a NAK line and sideband pack data
# Extract the pack data from sideband by parsing raw pkt-lines
pack_data = extract_sideband_pack(response)
assert pack_data != nil
assert byte_size(pack_data) > 0
# Verify the pack is parseable
assert {:ok, entries} = ExGitObjectstore.Pack.Reader.parse(pack_data)
assert length(entries) >= 3
types = Enum.map(entries, & &1.type) |> MapSet.new()
assert :blob in types
assert :tree in types
assert :commit in types
end
end
describe "feed - fetch (wants and haves)" do
test "sends only new objects" do
repo = RepoHelper.memory_repo()
ExGitObjectstore.init(repo)
# First commit (client already has this)
{c1_sha, _, _} = create_commit(repo, "v1\n", "first\n")
:ok = Ref.put(repo, "refs/heads/main", c1_sha, nil)
# Second commit (client wants this)
{c2_sha, _, _} = create_commit(repo, "v2\n", "second\n", [c1_sha])
:ok = Ref.put(repo, "refs/heads/main", c2_sha, c1_sha)
{_advert, state} = UploadPack.init(repo)
# Client has c1, wants c2
client_data =
PktLine.encode("want #{c2_sha}") <>
PktLine.encode("have #{c1_sha}") <>
PktLine.flush() <>
PktLine.encode("done")
{response, state} = UploadPack.feed(state, client_data)
assert UploadPack.done?(state)
assert byte_size(response) > 0
end
end
# Extract pack data from sideband-encoded pkt-line response.
# The response contains NAK text lines and sideband data lines (band byte 1 = pack).
# PktLine.decode strips newlines from data, so we need raw parsing for sideband.
defp extract_sideband_pack(response) do
extract_sideband_loop(response, <<>>)
end
defp extract_sideband_loop(<<>>, acc), do: if(acc == <<>>, do: nil, else: acc)
defp extract_sideband_loop(<<"0000", _rest::binary>>, acc) do
# Flush — stop
if acc == <<>>, do: nil, else: acc
end
defp extract_sideband_loop(<<hex::binary-size(4), rest::binary>>, acc) do
case Integer.parse(hex, 16) do
{len, ""} when len >= 5 ->
payload_len = len - 4
if byte_size(rest) >= payload_len do
<<payload::binary-size(payload_len), after_pkt::binary>> = rest
case payload do
<<1, pack_chunk::binary>> ->
# Sideband band 1 — pack data
extract_sideband_loop(after_pkt, acc <> pack_chunk)
_ ->
# NAK or other text line, skip
extract_sideband_loop(after_pkt, acc)
end
else
if acc == <<>>, do: nil, else: acc
end
_ ->
if acc == <<>>, do: nil, else: acc
end
end
defp extract_sideband_loop(_, acc), do: if(acc == <<>>, do: nil, else: acc)
end