|
|
CHANGELOG.md
|
+54
−0
|
@@ -7,6 +7,60 @@
## [Unreleased]
### Added
- **Git LFS support (spec v1).** Full Large File Storage implementation exposed as pure request/response modules, matching the existing `UploadPack`/`ReceivePack` style (no HTTP server in-tree). - `ExGitObjectstore.Lfs.Pointer` — parse and emit spec-compliant pointer blobs with strict validation (version-first, alphabetical key order, sha256-only OIDs, canonical decimal size, trailing LF). - `ExGitObjectstore.Lfs.Store` — behaviour parallel to `Storage`, keyed by SHA256. Streaming `put/4` verifies the observed SHA256 matches the claimed OID and discards the write on mismatch. Shared conformance test suite at `ExGitObjectstore.Test.LfsStoreConformance`. - Backends: `Lfs.Store.Memory`, `Lfs.Store.Filesystem`, `Lfs.Store.S3`. S3 uses multipart upload for streaming PUT and exposes optional `presigned_upload/5` / `presigned_download/4` callbacks for direct-to-S3 client transfers. - `ExGitObjectstore.Lfs.Batch` — Batch API handler (`POST /objects/batch`) returning spec-compliant upload / download / verify actions. Uses presigned URLs when the backend supports them; falls back to library-served URLs for Filesystem/Memory. - `ExGitObjectstore.Lfs.Transfer` — basic-transfer handlers for `GET/PUT /objects/:oid` and `POST /objects/:oid/verify`, with streaming downloads and SHA256-verified uploads. - `ExGitObjectstore.Lfs.Locks` — Locks API v1: create, list, verify, unlock (with `force` for admin override). Lock metadata stored on the repo's `Storage` backend under `lfs/locks/*.json`. - Telemetry spans emitted at `[:ex_git_objectstore, :lfs, :batch | :transfer | :lock]`. - `Repo` gains optional `:lfs_storage` option alongside `:storage`. - End-to-end interop coverage against the real `git lfs` binary via a Bandit-backed test HTTP adapter. 11 scenarios exercise push, smudge (download), idempotent re-push, edge-case payload sizes (0-byte via direct HTTP, 1-byte via the CLI), multi-file batch pushes, mixed-state batches (present + absent), concurrent parallel uploads (10 files with `lfs.concurrenttransfers=8`), direct-HTTP OID tampering (server must 422 and leave nothing on disk), lock create/list/verify/unlock with conflict and 403-by -non-owner paths, and a full end-to-end `git push` → `git clone` → `git lfs pull` roundtrip over smart-http. Found and fixed one real bug: `Batch.handle/3` was double-prefixing `repos/<id>/lfs` into action URLs — the `:base_url` is now the LFS root itself and the module emits `<base_url>/objects/<oid>` and `<base_url>/verify`. The test adapter also wires the existing `UploadPack`/`ReceivePack` modules to the git smart-http v0 endpoints (`GET /info/refs`, `POST /git-upload-pack`, `POST /git-receive-pack`) so a real `git clone` can complete the full clone-then-lfs-pull flow. - S3 backend interop coverage: 14 conformance tests against real MinIO plus 2 end-to-end `git lfs push`/`smudge` scenarios that exercise the presigned-URL path — client uploads directly to MinIO via presigned PUTs and downloads via presigned GETs, with the library only mediating batch + verify.
### Fixed
- **UploadPackV2: omit `acknowledgments` section when the client sends
|
|
|
lib/ex_git_objectstore/lfs/batch.ex
|
+240
−0
|
@@ -1,0 +1,240 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Batch do @moduledoc """ Git LFS Batch API handler (spec v1).
Pure request/response function — the caller wires this to an HTTP endpoint at `POST /objects/batch`. Returns a map suitable for JSON encoding.
## Request (decoded JSON)
%{ "operation" => "upload" | "download", "transfers" => ["basic", ...], "objects" => [%{"oid" => <hex>, "size" => <int>}, ...], "hash_algo" => "sha256" }
## Response
%{ "transfer" => "basic", "objects" => [ %{"oid" => ..., "size" => ..., "actions" => %{...}} | %{"oid" => ..., "size" => ..., "error" => %{"code" => ..., "message" => ...}} ], "hash_algo" => "sha256" } """
alias ExGitObjectstore.Lfs.Store alias ExGitObjectstore.Repo
@type response :: %{required(:status) => pos_integer(), required(:body) => map()}
@doc """ Handle a parsed batch request.
`opts`: * `:base_url` — (required for Filesystem/Memory backends) absolute URL of the LFS server root for this repo, e.g. `"https://git.example.com/myrepo/info/lfs"`. Library-served URLs are generated as `<base_url>/objects/<oid>` and `<base_url>/verify` so the consumer's router decides the URL shape. * `:expires_in` — seconds; forwarded to presigned-URL backends. """ @spec handle(Repo.t(), map(), keyword()) :: response() def handle(%Repo{} = repo, request, opts \\ []) when is_map(request) do with :ok <- validate_lfs_configured(repo), {:ok, op} <- parse_operation(request), {:ok, transfers} <- parse_transfers(request), {:ok, objects} <- parse_objects(request) do responses = Enum.map(objects, fn obj -> build_action(repo, op, obj, opts) end)
{:emit_telemetry, %{ status: 200, body: %{ "transfer" => select_transfer(transfers), "objects" => responses, "hash_algo" => "sha256" } }} |> emit() else {:error, :lfs_not_configured} -> error_response(501, "LFS not configured on this server")
{:error, {:invalid, msg}} -> error_response(422, msg) end end
defp emit({:emit_telemetry, response}) do :telemetry.execute( [:ex_git_objectstore, :lfs, :batch], %{object_count: length(response.body["objects"])}, %{operation: response.body["transfer"]} )
response end
defp validate_lfs_configured(%Repo{lfs_storage: nil}), do: {:error, :lfs_not_configured} defp validate_lfs_configured(%Repo{lfs_storage: {_, _}}), do: :ok
defp parse_operation(%{"operation" => op}) when op in ["upload", "download"], do: {:ok, op} defp parse_operation(_), do: {:error, {:invalid, "missing or invalid 'operation'"}}
defp parse_transfers(%{"transfers" => list}) when is_list(list) and list != [] do if "basic" in list, do: {:ok, list}, else: {:error, {:invalid, "no supported transfer"}} end
defp parse_transfers(_), do: {:ok, ["basic"]}
defp parse_objects(%{"objects" => objs}) when is_list(objs) and objs != [] do objs |> Enum.reduce_while({:ok, []}, fn obj, {:ok, acc} -> case parse_object(obj) do {:ok, parsed} -> {:cont, {:ok, [parsed | acc]}} {:error, _} = err -> {:halt, err} end end) |> case do {:ok, acc} -> {:ok, Enum.reverse(acc)} err -> err end end
defp parse_objects(_), do: {:error, {:invalid, "missing 'objects'"}}
defp parse_object(%{"oid" => oid, "size" => size}) when is_binary(oid) and is_integer(size) and size >= 0 do case Store.validate_oid(oid) do :ok -> {:ok, %{oid: oid, size: size}} _ -> {:error, {:invalid, "invalid oid: #{oid}"}} end end
defp parse_object(_), do: {:error, {:invalid, "invalid object entry"}}
defp select_transfer(transfers), do: if("basic" in transfers, do: "basic", else: hd(transfers))
defp build_action(repo, "download", %{oid: oid, size: size} = _obj, opts) do base = %{"oid" => oid, "size" => size} {mod, cfg} = repo.lfs_storage prefix = Store.prefix(repo)
case mod.stat(cfg, prefix, oid) do {:ok, %{size: actual}} when actual == size -> href_spec = download_url(mod, cfg, prefix, oid, repo, opts) Map.merge(base, %{"authenticated" => true, "actions" => %{"download" => href_spec}})
{:ok, %{size: actual}} -> Map.put(base, "error", %{ "code" => 422, "message" => "size mismatch: expected #{size}, stored #{actual}" })
{:error, :not_found} -> Map.put(base, "error", %{"code" => 404, "message" => "object not found"})
{:error, r} -> Map.put(base, "error", %{"code" => 500, "message" => "storage error: #{inspect(r)}"}) end end
defp build_action(repo, "upload", %{oid: oid, size: size} = _obj, opts) do base = %{"oid" => oid, "size" => size} {mod, cfg} = repo.lfs_storage prefix = Store.prefix(repo)
if mod.exists?(cfg, prefix, oid) do # Already present — no action needed, client skips upload. Map.put(base, "authenticated", true) else upload = upload_url(mod, cfg, prefix, oid, size, repo, opts) verify = verify_url(repo, opts)
Map.merge(base, %{ "authenticated" => true, "actions" => %{"upload" => upload, "verify" => verify} }) end end
defp download_url(mod, cfg, prefix, oid, repo, opts) do if function_exported?(mod, :presigned_download, 4) do case mod.presigned_download(cfg, prefix, oid, opts) do {:ok, spec} -> url_spec_to_map(spec) {:error, _} -> library_served_download(repo, oid, opts) end else library_served_download(repo, oid, opts) end end
defp upload_url(mod, cfg, prefix, oid, size, repo, opts) do if function_exported?(mod, :presigned_upload, 5) do case mod.presigned_upload(cfg, prefix, oid, size, opts) do {:ok, spec} -> url_spec_to_map(spec) {:error, _} -> library_served_upload(repo, oid, opts) end else library_served_upload(repo, oid, opts) end end
defp verify_url(_repo, opts) do base_url = Keyword.fetch!(opts, :base_url)
%{ "href" => "#{base_url}/verify", "expires_in" => expires_in(opts) } end
defp library_served_download(_repo, oid, opts) do base_url = Keyword.fetch!(opts, :base_url)
%{ "href" => "#{base_url}/objects/#{oid}", "expires_in" => expires_in(opts) } end
defp library_served_upload(_repo, oid, opts) do base_url = Keyword.fetch!(opts, :base_url)
%{ "href" => "#{base_url}/objects/#{oid}", "header" => %{}, "expires_in" => expires_in(opts) } end
defp url_spec_to_map(%{href: href, headers: headers, expires_in: expires_in}) do %{"href" => href, "header" => headers, "expires_in" => expires_in} end
defp expires_in(opts), do: Keyword.get(opts, :expires_in, 3_600)
defp error_response(status, message) do %{status: status, body: %{"message" => message}} end end
|
|
|
lib/ex_git_objectstore/lfs/locks.ex
|
+271
−0
|
@@ -1,0 +1,271 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Locks do @moduledoc """ Git LFS Locks API v1.
Locks are stored as JSON blobs on the repo's `Storage` backend under the key prefix `lfs/locks/<id>.json`. This keeps LFS content blobs (huge, content-addressed) separate from lock metadata (tiny, path- addressed).
## Concurrency
The underlying `Storage.put_blob/4` is not atomic across replicas. For single-writer workloads (one server instance, or coordinator- fronted multi-writer), path-uniqueness is enforced via a list-then- write pattern. Concurrent writers can race and both succeed; the resulting duplicate can be cleaned up by an operator running `list/2` and `unlock/3` with `force: true`.
## Endpoints
* `POST /locks` → `create/3` * `GET /locks` → `list/2` * `POST /locks/verify` → `verify/3` * `POST /locks/:id/unlock` → `unlock/4` """
alias ExGitObjectstore.Repo
@type lock :: %{ required(:id) => String.t(), required(:path) => String.t(), required(:locked_at) => String.t(), required(:owner) => %{required(:name) => String.t()} }
@blob_prefix "lfs/locks"
@doc """ Create a lock on `path` for `owner`.
Returns `{:ok, lock}` or `{:error, {:conflict, existing_lock}}` if `path` is already locked. """ @spec create(Repo.t(), String.t(), String.t()) :: {:ok, lock()} | {:error, {:conflict, lock()} | atom()} def create(%Repo{} = repo, path, owner_name) when is_binary(path) and path != "" and is_binary(owner_name) do :telemetry.span( [:ex_git_objectstore, :lfs, :lock], %{operation: :create, repo: repo.id, path: path}, fn -> result = do_create(repo, path, owner_name) {result, %{operation: :create, path: path, outcome: outcome(result)}} end ) end
def create(_, _, _), do: {:error, :bad_request}
defp do_create(repo, path, owner_name) do case find_by_path(repo, path) do {:ok, existing} -> {:error, {:conflict, existing}}
:none -> lock = %{ id: new_id(), path: path, locked_at: now_iso8601(), owner: %{name: owner_name} }
case write_lock(repo, lock) do :ok -> {:ok, lock} {:error, _} = err -> err end end end
@doc """ List all locks, optionally filtered by `path` or `id`. """ @spec list(Repo.t(), keyword()) :: {:ok, [lock()]} | {:error, atom()} def list(%Repo{} = repo, opts \\ []) do case list_all(repo) do {:ok, locks} -> filtered = locks |> filter_by(:path, Keyword.get(opts, :path)) |> filter_by(:id, Keyword.get(opts, :id))
{:ok, filtered}
err -> err end end
@doc """ Verify which locks are held by `owner_name` (ours) vs others (theirs). """ @spec verify(Repo.t(), String.t()) :: {:ok, %{ours: [lock()], theirs: [lock()]}} | {:error, atom()} def verify(%Repo{} = repo, owner_name) when is_binary(owner_name) do case list_all(repo) do {:ok, locks} -> {ours, theirs} = Enum.split_with(locks, fn l -> l.owner.name == owner_name end) {:ok, %{ours: ours, theirs: theirs}}
err -> err end end
@doc """ Release a lock by id. Non-owners must pass `force: true`. """ @spec unlock(Repo.t(), String.t(), String.t(), keyword()) :: {:ok, lock()} | {:error, atom()} def unlock(%Repo{} = repo, id, requester_name, opts \\ []) do :telemetry.span( [:ex_git_objectstore, :lfs, :lock], %{operation: :unlock, repo: repo.id, id: id}, fn -> result = do_unlock(repo, id, requester_name, opts) {result, %{operation: :unlock, id: id, outcome: outcome(result)}} end ) end
defp do_unlock(repo, id, requester_name, opts) do case read_lock(repo, id) do {:ok, lock} -> force? = Keyword.get(opts, :force, false)
cond do lock.owner.name == requester_name -> delete_and_return(repo, id, lock)
force? -> delete_and_return(repo, id, lock)
true -> {:error, :forbidden} end
{:error, :not_found} -> {:error, :not_found}
err -> err end end
# -- Private --
defp blob_key(id), do: "#{@blob_prefix}/#{id}.json"
defp new_id do :crypto.strong_rand_bytes(16) |> Base.encode16(case: :lower) end
defp now_iso8601 do DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_iso8601() end
defp read_lock(repo, id) do case Repo.storage_call(repo, :get_blob, [blob_key(id)]) do {:ok, data} -> {:ok, decode(data)} err -> err end end
defp find_by_path(repo, path) do with {:ok, locks} <- list_all(repo), %{} = found <- Enum.find(locks, fn l -> l.path == path end) do {:ok, found} else nil -> :none {:error, _} -> :none end end
defp list_all(%Repo{} = repo) do # Enumerate via storage.list_objects-style — but Storage doesn't # expose list_blobs. We list via a side-index blob that records all # lock IDs, updated on create/unlock. Single-writer assumption. case Repo.storage_call(repo, :get_blob, ["lfs/locks/_index.json"]) do {:ok, data} -> ids = Jason.decode!(data) locks = Enum.flat_map(ids, &safe_read(repo, &1)) {:ok, locks}
{:error, :not_found} -> {:ok, []}
err -> err end end
defp safe_read(repo, id) do case read_lock(repo, id) do {:ok, l} -> [l] _ -> [] end end
defp write_lock(repo, lock) when is_map(lock) do with :ok <- Repo.storage_call(repo, :put_blob, [blob_key(lock.id), encode(lock)]) do update_index(repo, &[lock.id | &1]) end end
defp delete_and_return(repo, id, lock) do with :ok <- Repo.storage_call(repo, :delete_blob, [blob_key(id)]), :ok <- update_index(repo, &List.delete(&1, id)) do {:ok, lock} end end
defp update_index(repo, fun) do ids = case Repo.storage_call(repo, :get_blob, ["lfs/locks/_index.json"]) do {:ok, data} -> Jason.decode!(data) _ -> [] end
new_ids = fun.(ids) |> Enum.uniq() Repo.storage_call(repo, :put_blob, ["lfs/locks/_index.json", Jason.encode!(new_ids)]) end
defp encode(lock), do: Jason.encode!(lock)
defp decode(data) do raw = Jason.decode!(data)
%{ id: raw["id"], path: raw["path"], locked_at: raw["locked_at"], owner: %{name: raw["owner"]["name"]} } end
defp filter_by(list, _k, nil), do: list defp filter_by(list, :path, v), do: Enum.filter(list, fn l -> l.path == v end) defp filter_by(list, :id, v), do: Enum.filter(list, fn l -> l.id == v end)
defp outcome(:ok), do: :ok defp outcome({:ok, _}), do: :ok defp outcome({:error, reason}), do: inspect_reason(reason)
defp inspect_reason({:conflict, _}), do: :conflict defp inspect_reason(r) when is_atom(r), do: r defp inspect_reason(other), do: inspect(other) end
|
|
|
lib/ex_git_objectstore/lfs/pointer.ex
|
+231
−0
|
@@ -1,0 +1,231 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Pointer do @moduledoc """ Parser and emitter for Git LFS pointer blobs (spec v1).
Pointer format:
version https://git-lfs.github.com/spec/v1 oid sha256:<64 hex> size <non-negative integer>
Additional keys may appear between `version` and `oid`, sorted alphabetically. Lines are terminated by a single LF. The pointer blob must end with an LF. """
@version_line "version https://git-lfs.github.com/spec/v1" @oid_hex_pattern ~r/\A[0-9a-f]{64}\z/ @size_decimal_pattern ~r/\A(?:0|[1-9][0-9]*)\z/ @key_pattern ~r/\A[a-z0-9.\-]+\z/ @max_pointer_bytes 4096
@type t :: %__MODULE__{ oid: String.t(), size: non_neg_integer(), extra: %{optional(String.t()) => String.t()} }
defstruct [:oid, :size, extra: %{}]
@doc """ Construct a new pointer from oid (sha256 hex) and size. """ @spec new(String.t(), integer(), %{optional(String.t()) => String.t()}) :: {:ok, t()} | {:error, atom()} def new(oid, size, extra \\ %{})
def new(oid, _size, _extra) when not is_binary(oid), do: {:error, :bad_oid}
def new(oid, size, extra) do with :ok <- validate_oid(oid), :ok <- validate_size(size), :ok <- validate_extra(extra) do {:ok, %__MODULE__{oid: oid, size: size, extra: extra}} end end
@doc """ Parse a pointer blob. Returns `{:ok, pointer}` or `{:error, reason}`. """ @spec parse(binary()) :: {:ok, t()} | {:error, atom()} def parse(blob) when is_binary(blob) do cond do byte_size(blob) > @max_pointer_bytes -> {:error, :too_large}
blob == "" -> {:error, :empty}
not String.ends_with?(blob, "\n") -> {:error, :missing_trailing_lf}
true -> do_parse(blob) end end
def parse(_), do: {:error, :not_binary}
@doc """ Returns true if the blob looks like a well-formed LFS pointer. Cheap precheck; equivalent to `match?({:ok, _}, parse/1)`. """ @spec pointer?(binary()) :: boolean() def pointer?(blob) when is_binary(blob) do match?({:ok, _}, parse(blob)) end
def pointer?(_), do: false
@doc """ Emit a pointer back to canonical pointer-blob form. """ @spec emit(t()) :: binary() def emit(%__MODULE__{oid: oid, size: size, extra: extra}) do extras = extra |> Enum.sort_by(fn {k, _} -> k end) |> Enum.map(fn {k, v} -> [k, ?\s, v, ?\n] end)
IO.iodata_to_binary([ @version_line, ?\n, extras, "oid sha256:", oid, ?\n, "size ", Integer.to_string(size), ?\n ]) end
# -- Internal --
defp do_parse(blob) do # Strict LF-only split. If a CR sneaks in, a key or value will fail validation. lines = String.split(blob, "\n")
# Because blob ends with \n, last element is "". Drop it. case List.pop_at(lines, -1) do {"", content_lines} -> parse_lines(content_lines) _ -> {:error, :bad_line} end end
defp parse_lines([]), do: {:error, :empty}
defp parse_lines([first | rest]) do case find_version_position([first | rest]) do :first -> parse_after_version(rest) :elsewhere -> {:error, :version_not_first} :missing -> {:error, :bad_version} end end
defp find_version_position([@version_line | _]), do: :first
defp find_version_position(lines) do if Enum.any?(lines, &(&1 == @version_line)) do :elsewhere else :missing end end
defp parse_after_version(lines) do with {:ok, pairs} <- split_each(lines, []), :ok <- validate_keys_sorted_unique(pairs), {:ok, map} <- pairs_to_map(pairs), {:ok, oid} <- extract_oid(map), {:ok, size} <- extract_size(map) do extra = map |> Map.delete("oid") |> Map.delete("size") {:ok, %__MODULE__{oid: oid, size: size, extra: extra}} end end
defp split_each([], acc), do: {:ok, Enum.reverse(acc)}
defp split_each([line | rest], acc) do case String.split(line, " ", parts: 2) do [key, value] -> cond do not Regex.match?(@key_pattern, key) -> {:error, :bad_key} value == "" -> {:error, :empty_value} true -> split_each(rest, [{key, value} | acc]) end
_ -> {:error, :bad_line} end end
defp validate_keys_sorted_unique(pairs) do keys = Enum.map(pairs, fn {k, _} -> k end) sorted = Enum.sort(keys)
cond do length(Enum.uniq(keys)) != length(keys) -> {:error, :duplicate_key} keys != sorted -> {:error, :keys_unsorted} true -> :ok end end
defp pairs_to_map(pairs), do: {:ok, Map.new(pairs)}
defp extract_oid(%{"oid" => "sha256:" <> hex}) do if Regex.match?(@oid_hex_pattern, hex), do: {:ok, hex}, else: {:error, :bad_oid} end
defp extract_oid(%{"oid" => _}), do: {:error, :bad_oid} defp extract_oid(_), do: {:error, :missing_oid}
defp extract_size(%{"size" => raw}) do if Regex.match?(@size_decimal_pattern, raw) do {:ok, String.to_integer(raw)} else {:error, :bad_size} end end
defp extract_size(_), do: {:error, :missing_size}
defp validate_oid(oid) when is_binary(oid) do if Regex.match?(@oid_hex_pattern, oid), do: :ok, else: {:error, :bad_oid} end
defp validate_oid(_), do: {:error, :bad_oid}
defp validate_size(size) when is_integer(size) and size >= 0, do: :ok defp validate_size(_), do: {:error, :bad_size}
defp validate_extra(extra) when is_map(extra) do Enum.reduce_while(extra, :ok, fn {k, v}, _ -> cond do not is_binary(k) or not is_binary(v) -> {:halt, {:error, :bad_extra}} k in ["version", "oid", "size"] -> {:halt, {:error, :reserved_key}} not Regex.match?(@key_pattern, k) -> {:halt, {:error, :bad_key}} v == "" -> {:halt, {:error, :empty_value}} true -> {:cont, :ok} end end) end
defp validate_extra(_), do: {:error, :bad_extra} end
|
|
|
lib/ex_git_objectstore/lfs/store.ex
|
+149
−0
|
@@ -1,0 +1,149 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store do @moduledoc """ Behaviour for pluggable Git LFS object storage backends.
Parallel to `ExGitObjectstore.Storage` but keyed by SHA256 OIDs and scoped to LFS content — large binary blobs referenced by LFS pointer files in the regular git object store.
All callbacks receive a backend-specific `config` map and a `prefix` (repo-scoped key prefix like `"repos/<id>/lfs"`).
## Streaming
`put/4` accepts a chunk stream (iolist-compatible `Enumerable`). The backend MUST compute the SHA256 of the stream while writing and MUST reject and discard the write if the observed digest does not equal the claimed `oid`.
`get/3` returns `{size, stream}` for chunked reads. Callers must be able to iterate the stream without holding the entire object in memory.
## Presigned URLs
Optional — only S3 (or other remote blob stores) implement these. Filesystem and Memory should return `{:error, :not_supported}`. """
@type config :: map() @type prefix :: String.t() @type oid :: String.t() @type size :: non_neg_integer() @type chunk :: binary() | iolist() @type reason :: atom() | {atom(), term()}
@type url_spec :: %{ required(:href) => String.t(), required(:headers) => %{optional(String.t()) => String.t()}, required(:expires_in) => pos_integer() }
# -- Object operations --
@doc """ Stream bytes into the backend under `oid`, verifying the observed SHA256 matches `oid`. Returns `{:ok, bytes_written}` on success or `{:error, :oid_mismatch}` (and discards the partial write) on failure. """ @callback put(config, prefix, oid, Enumerable.t()) :: {:ok, size} | {:error, reason}
@doc """ Fetch an object. Returns its size and a chunked read stream. """ @callback get(config, prefix, oid) :: {:ok, %{size: size, stream: Enumerable.t()}} | {:error, reason}
@callback exists?(config, prefix, oid) :: boolean()
@callback stat(config, prefix, oid) :: {:ok, %{size: size}} | {:error, reason}
@callback delete(config, prefix, oid) :: :ok | {:error, reason}
@callback list(config, prefix) :: {:ok, [oid]} | {:error, reason}
# -- Presigned URL operations (optional) --
@callback presigned_upload(config, prefix, oid, size, opts :: keyword()) :: {:ok, url_spec()} | {:error, reason}
@callback presigned_download(config, prefix, oid, opts :: keyword()) :: {:ok, url_spec()} | {:error, reason}
@optional_callbacks [presigned_upload: 5, presigned_download: 4]
@oid_pattern ~r/\A[0-9a-f]{64}\z/
@doc """ Validate a claimed OID (64 lowercase hex chars). """ @spec validate_oid(term()) :: :ok | {:error, :bad_oid} def validate_oid(oid) when is_binary(oid) do if Regex.match?(@oid_pattern, oid), do: :ok, else: {:error, :bad_oid} end
def validate_oid(_), do: {:error, :bad_oid}
@doc """ Dispatch a call through `Repo`'s configured LFS backend. """ @spec call(ExGitObjectstore.Repo.t(), atom(), [term()]) :: term() def call(%ExGitObjectstore.Repo{lfs_storage: nil}, _fun, _args), do: {:error, :lfs_not_configured}
def call(%ExGitObjectstore.Repo{lfs_storage: {mod, cfg}} = repo, fun, args) do apply(mod, fun, [cfg, prefix(repo) | args]) end
@doc """ Storage key prefix for LFS content for a given repo. """ @spec prefix(ExGitObjectstore.Repo.t()) :: String.t() def prefix(%ExGitObjectstore.Repo{id: id}), do: "repos/#{id}/lfs"
@doc """ Run `fun` over each chunk of `enum`, computing SHA256 and byte count.
`fun` receives each raw binary chunk. If it returns `{:error, r}` the iteration halts and the error is returned. On success returns `{:ok, bytes, hex_digest}`. """ @spec hash_while(Enumerable.t(), (binary() -> :ok | {:error, reason})) :: {:ok, size(), String.t()} | {:error, reason} def hash_while(enum, fun) when is_function(fun, 1) do Enum.reduce_while(enum, {:ok, 0, :crypto.hash_init(:sha256)}, fn chunk, {:ok, bytes, ctx} -> bin = IO.iodata_to_binary(chunk)
case fun.(bin) do :ok -> {:cont, {:ok, bytes + byte_size(bin), :crypto.hash_update(ctx, bin)}}
{:error, _} = err -> {:halt, err} end end) |> case do {:ok, bytes, ctx} -> digest = :crypto.hash_final(ctx) |> Base.encode16(case: :lower) {:ok, bytes, digest}
{:error, _} = err -> err end end end
|
|
|
lib/ex_git_objectstore/lfs/store/filesystem.ex
|
+202
−0
|
@@ -1,0 +1,202 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.Filesystem do @moduledoc """ Local filesystem LFS store backend.
Layout under `<root>/<prefix>/`:
<oid[0..1]>/<oid[2..3]>/<full-oid>
Config: `%{root: "/path/to/storage"}`. """
@behaviour ExGitObjectstore.Lfs.Store
alias ExGitObjectstore.Lfs.Store
@chunk_size 64 * 1024
@impl true def put(config, prefix, oid, stream) do with :ok <- Store.validate_oid(oid) do path = object_path(config, prefix, oid) dir = Path.dirname(path) File.mkdir_p!(dir) tmp = path <> ".tmp.#{:erlang.unique_integer([:positive])}"
case :file.open(tmp, [:write, :binary, :raw]) do {:ok, fd} -> write_and_verify(fd, tmp, path, oid, stream)
{:error, reason} -> {:error, reason} end end end
defp write_and_verify(fd, tmp, path, oid, stream) do try do Store.hash_while(stream, fn bin -> case :file.write(fd, bin) do :ok -> :ok {:error, r} -> {:error, r} end end) else {:ok, bytes, ^oid} -> :file.close(fd)
case File.rename(tmp, path) do :ok -> {:ok, bytes}
{:error, r} -> _ = File.rm(tmp) {:error, r} end
{:ok, _bytes, _other} -> :file.close(fd) _ = File.rm(tmp) {:error, :oid_mismatch}
{:error, _} = err -> :file.close(fd) _ = File.rm(tmp) err after :ok end rescue e -> _ = File.rm(tmp) {:error, {:write_failed, Exception.message(e)}} end
@impl true def get(config, prefix, oid) do with :ok <- Store.validate_oid(oid) do path = object_path(config, prefix, oid)
case File.stat(path) do {:ok, %{size: size}} -> stream = File.stream!(path, @chunk_size) {:ok, %{size: size, stream: stream}}
{:error, :enoent} -> {:error, :not_found}
{:error, r} -> {:error, r} end end end
@impl true def exists?(config, prefix, oid) do case Store.validate_oid(oid) do :ok -> File.regular?(object_path(config, prefix, oid)) _ -> false end end
@impl true def stat(config, prefix, oid) do with :ok <- Store.validate_oid(oid) do case File.stat(object_path(config, prefix, oid)) do {:ok, %{size: size}} -> {:ok, %{size: size}} {:error, :enoent} -> {:error, :not_found} {:error, r} -> {:error, r} end end end
@impl true def delete(config, prefix, oid) do with :ok <- Store.validate_oid(oid) do case File.rm(object_path(config, prefix, oid)) do :ok -> :ok {:error, :enoent} -> :ok {:error, r} -> {:error, r} end end end
@impl true def list(config, prefix) do base = safe_path(config.root, prefix)
case File.ls(base) do {:ok, top} -> oids = top |> Enum.filter(&fanout2?/1) |> Enum.flat_map(&list_second_level(base, &1)) |> Enum.sort()
{:ok, oids}
{:error, :enoent} -> {:ok, []}
{:error, r} -> {:error, r} end end
defp list_second_level(base, dir) do case File.ls(Path.join(base, dir)) do {:ok, subs} -> subs |> Enum.filter(&fanout2?/1) |> Enum.flat_map(&list_leaf(base, dir, &1))
_ -> [] end end
defp list_leaf(base, d1, d2) do case File.ls(Path.join([base, d1, d2])) do {:ok, files} -> Enum.filter(files, fn f -> byte_size(f) == 64 and Regex.match?(~r/\A[0-9a-f]{64}\z/, f) end)
_ -> [] end end
defp fanout2?(<<a, b>>) when a in ?0..?9 or a in ?a..?f, do: b in ?0..?9 or b in ?a..?f defp fanout2?(_), do: false
defp object_path(config, prefix, oid) do <<a::binary-size(2), b::binary-size(2), _::binary>> = oid safe_path(config.root, Path.join([prefix, a, b, oid])) end
defp safe_path(root, relative) do full = Path.join(root, relative) |> Path.expand() root_expanded = Path.expand(root)
if String.starts_with?(full, root_expanded <> "/") or full == root_expanded do full else raise ArgumentError, "path traversal detected: #{relative}" end end end
|
|
|
lib/ex_git_objectstore/lfs/store/memory.ex
|
+135
−0
|
@@ -1,0 +1,135 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.Memory do @moduledoc """ In-memory LFS store backend for testing.
Config: `%{pid: pid}` referencing an Agent started via `start_link/0`. """
@behaviour ExGitObjectstore.Lfs.Store
alias ExGitObjectstore.Lfs.Store
@chunk_size 64 * 1024
@spec start_link() :: {:ok, pid()} def start_link do Agent.start_link(fn -> %{} end) end
@spec config(pid()) :: map() def config(pid), do: %{pid: pid}
@impl true def put(%{pid: pid}, prefix, oid, stream) do with :ok <- Store.validate_oid(oid) do {chunks, hash} = collect(stream) commit_if_match(pid, prefix, oid, chunks, hash) end end
defp collect(stream) do Enum.reduce(stream, {[], :crypto.hash_init(:sha256)}, fn chunk, {acc, ctx} -> bin = IO.iodata_to_binary(chunk) {[bin | acc], :crypto.hash_update(ctx, bin)} end) end
defp commit_if_match(pid, prefix, oid, chunks, ctx) do observed = :crypto.hash_final(ctx) |> Base.encode16(case: :lower)
if observed == oid do data = chunks |> Enum.reverse() |> IO.iodata_to_binary() Agent.update(pid, &Map.put(&1, key(prefix, oid), data)) {:ok, byte_size(data)} else {:error, :oid_mismatch} end end
@impl true def get(%{pid: pid}, prefix, oid) do with :ok <- Store.validate_oid(oid) do case Agent.get(pid, &Map.get(&1, key(prefix, oid))) do nil -> {:error, :not_found}
data -> stream = chunk_binary(data) {:ok, %{size: byte_size(data), stream: stream}} end end end
@impl true def exists?(%{pid: pid}, prefix, oid) do case Store.validate_oid(oid) do :ok -> Agent.get(pid, &Map.has_key?(&1, key(prefix, oid))) _ -> false end end
@impl true def stat(%{pid: pid}, prefix, oid) do with :ok <- Store.validate_oid(oid) do case Agent.get(pid, &Map.get(&1, key(prefix, oid))) do nil -> {:error, :not_found} data -> {:ok, %{size: byte_size(data)}} end end end
@impl true def delete(%{pid: pid}, prefix, oid) do with :ok <- Store.validate_oid(oid) do Agent.update(pid, &Map.delete(&1, key(prefix, oid))) :ok end end
@impl true def list(%{pid: pid}, prefix) do full_prefix = prefix <> "/"
oids = Agent.get(pid, fn state -> state |> Map.keys() |> Enum.filter(&String.starts_with?(&1, full_prefix)) |> Enum.map(&(&1 |> String.replace_prefix(full_prefix, "") |> String.replace("/", ""))) |> Enum.sort() end)
{:ok, oids} end
defp key(prefix, oid), do: "#{prefix}/#{fanout(oid)}"
defp fanout(<<a::binary-size(2), b::binary-size(2), rest::binary>>), do: "#{a}/#{b}/#{rest}"
defp chunk_binary(<<>>), do: []
defp chunk_binary(data) when byte_size(data) <= @chunk_size, do: [data]
defp chunk_binary(data) do Stream.unfold(data, fn <<>> -> nil bin when byte_size(bin) <= @chunk_size -> {bin, <<>>} <<head::binary-size(@chunk_size), rest::binary>> -> {head, rest} end) end end
|
|
|
lib/ex_git_objectstore/lfs/store/s3.ex
|
+368
−0
|
@@ -1,0 +1,368 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.S3 do @moduledoc """ S3-compatible LFS store backend.
Key layout: `<prefix>/<oid[0..1]>/<oid[2..3]>/<full-oid>`.
## Transfer modes
This backend supports two transfer modes:
1. **Direct (library-served)** — `put/4` and `get/3` stream bytes through the Elixir process. Useful for small-to-medium objects or when the client cannot reach S3 directly. `put/4` uses S3 multipart upload and verifies the SHA256 before completing; on mismatch, the upload is aborted.
2. **Presigned URLs** — `presigned_upload/5` and `presigned_download/4` return short-lived URLs the LFS client uploads to / downloads from S3 directly. For blobs over tens of megabytes, this mode is strongly preferred.
Verification of SHA256 in presigned mode happens at the verify endpoint (via `stat/3` confirming the object is present plus a size check).
## Config
%{ bucket: "my-bucket", ex_aws_config: [...], multipart_part_size: 16 * 1024 * 1024, # optional, default 16 MiB presign_expires_in: 3_600 # optional, default 1h } """
@behaviour ExGitObjectstore.Lfs.Store
alias ExGitObjectstore.Lfs.Store
@default_part_size 16 * 1024 * 1024 @default_expires 3_600
@impl true def put(config, prefix, oid, stream) do with :ok <- Store.validate_oid(oid) do key = object_key(prefix, oid) part_size = Map.get(config, :multipart_part_size, @default_part_size)
case initiate_multipart(config, key) do {:ok, upload_id} -> stream_multipart(config, key, upload_id, oid, stream, part_size)
{:error, _} = err -> err end end end
defp stream_multipart(config, key, upload_id, oid, stream, part_size) do state = %{ parts: [], buffer: <<>>, part_number: 1, ctx: :crypto.hash_init(:sha256), bytes: 0, config: config, key: key, upload_id: upload_id, part_size: part_size }
try do Enum.reduce(stream, state, &feed_chunk/2) |> flush_buffer() |> finalise_multipart(oid) catch kind, reason -> abort_multipart(config, key, upload_id) {:error, {kind, reason}} end end
defp feed_chunk(chunk, state) do bin = IO.iodata_to_binary(chunk) new_buf = state.buffer <> bin
new_state = %{ state | buffer: new_buf, ctx: :crypto.hash_update(state.ctx, bin), bytes: state.bytes + byte_size(bin) }
emit_full_parts(new_state) end
defp emit_full_parts(%{buffer: buf, part_size: ps} = state) when byte_size(buf) >= ps do <<part::binary-size(ps), rest::binary>> = buf {:ok, etag} = upload_part(state.config, state.key, state.upload_id, state.part_number, part)
%{ state | buffer: rest, parts: [{state.part_number, etag} | state.parts], part_number: state.part_number + 1 } |> emit_full_parts() end
defp emit_full_parts(state), do: state
defp flush_buffer(%{buffer: <<>>, parts: [_ | _]} = state), do: state
defp flush_buffer(%{buffer: buf} = state) do {:ok, etag} = upload_part(state.config, state.key, state.upload_id, state.part_number, buf) %{state | buffer: <<>>, parts: [{state.part_number, etag} | state.parts]} end
defp finalise_multipart(state, expected_oid) do observed = :crypto.hash_final(state.ctx) |> Base.encode16(case: :lower)
if observed == expected_oid do parts = Enum.reverse(state.parts)
case complete_multipart(state.config, state.key, state.upload_id, parts) do :ok -> {:ok, state.bytes} {:error, _} = err -> err end else abort_multipart(state.config, state.key, state.upload_id) {:error, :oid_mismatch} end end
@impl true def get(config, prefix, oid) do with :ok <- Store.validate_oid(oid), key = object_key(prefix, oid), {:ok, size} <- s3_head(config, key) do fetch_body(config, key, size) end end
# Library-served GET buffers the response. For large blobs, callers # should use presigned_download/4 and hand the URL to the LFS client. defp fetch_body(config, key, size) do op = ExAws.S3.get_object(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do {:ok, %{body: body}} -> {:ok, %{size: size, stream: [body]}} {:error, r} -> {:error, r} end end
@impl true def exists?(config, prefix, oid) do case Store.validate_oid(oid) do :ok -> match?({:ok, _}, s3_head(config, object_key(prefix, oid)))
_ -> false end end
@impl true def stat(config, prefix, oid) do with :ok <- Store.validate_oid(oid), {:ok, size} <- s3_head(config, object_key(prefix, oid)) do {:ok, %{size: size}} end end
@impl true def delete(config, prefix, oid) do with :ok <- Store.validate_oid(oid) do op = ExAws.S3.delete_object(config.bucket, object_key(prefix, oid))
case ExAws.request(op, ex_aws_config(config)) do {:ok, _} -> :ok {:error, r} -> {:error, r} end end end
@impl true def list(config, prefix) do list_all(config, prefix <> "/", nil, []) end
defp list_all(config, prefix, continuation_token, acc) do opts = [prefix: prefix] ++ if(continuation_token, do: [continuation_token: continuation_token], else: [])
op = ExAws.S3.list_objects_v2(config.bucket, opts)
case ExAws.request(op, ex_aws_config(config)) do {:ok, %{body: %{contents: c, is_truncated: "true", next_continuation_token: t}}} -> list_all(config, prefix, t, Enum.reverse(extract_oids(c, prefix)) ++ acc)
{:ok, %{body: %{contents: c}}} -> {:ok, Enum.reverse(Enum.reverse(extract_oids(c, prefix)) ++ acc)}
{:ok, %{body: _}} -> {:ok, Enum.reverse(acc)}
{:error, r} -> {:error, r} end end
defp extract_oids(contents, prefix) do contents |> Enum.map(& &1.key) |> Enum.flat_map(fn key -> case String.replace_prefix(key, prefix, "") |> String.split("/") do [_a, _b, oid] when byte_size(oid) == 64 -> [oid] _ -> [] end end) end
@impl true def presigned_upload(config, prefix, oid, size, opts) do with :ok <- Store.validate_oid(oid) do expires_in = Keyword.get(opts, :expires_in, Map.get(config, :presign_expires_in, @default_expires))
key = object_key(prefix, oid) query_params = [{"Content-Length", Integer.to_string(size)}]
case ExAws.S3.presigned_url( ex_aws_config_struct(config), :put, config.bucket, key, expires_in: expires_in, query_params: query_params ) do {:ok, url} -> {:ok, %{ href: url, headers: %{"Content-Length" => Integer.to_string(size)}, expires_in: expires_in }}
{:error, r} -> {:error, r} end end end
@impl true def presigned_download(config, prefix, oid, opts) do with :ok <- Store.validate_oid(oid) do expires_in = Keyword.get(opts, :expires_in, Map.get(config, :presign_expires_in, @default_expires))
key = object_key(prefix, oid)
case ExAws.S3.presigned_url( ex_aws_config_struct(config), :get, config.bucket, key, expires_in: expires_in ) do {:ok, url} -> {:ok, %{href: url, headers: %{}, expires_in: expires_in}} {:error, r} -> {:error, r} end end end
# -- S3 helpers --
defp object_key(prefix, oid) do <<a::binary-size(2), b::binary-size(2), _::binary>> = oid "#{prefix}/#{a}/#{b}/#{oid}" end
defp s3_head(config, key) do op = ExAws.S3.head_object(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do {:ok, %{headers: headers}} -> size = headers |> Enum.find_value(0, fn {"Content-Length", v} -> String.to_integer(v) {"content-length", v} -> String.to_integer(v) _ -> nil end)
{:ok, size}
{:error, {:http_error, 404, _}} -> {:error, :not_found}
{:error, r} -> {:error, r} end end
defp initiate_multipart(config, key) do op = ExAws.S3.initiate_multipart_upload(config.bucket, key)
case ExAws.request(op, ex_aws_config(config)) do {:ok, %{body: %{upload_id: id}}} -> {:ok, id} {:error, r} -> {:error, r} end end
defp upload_part(config, key, upload_id, part_number, body) do op = ExAws.S3.upload_part(config.bucket, key, upload_id, part_number, body)
case ExAws.request(op, ex_aws_config(config)) do {:ok, %{headers: headers}} -> etag = Enum.find_value(headers, fn {"ETag", v} -> v {"etag", v} -> v _ -> nil end)
{:ok, etag}
{:error, r} -> throw({:upload_part_failed, r}) end end
defp complete_multipart(config, key, upload_id, parts) do op = ExAws.S3.complete_multipart_upload(config.bucket, key, upload_id, parts)
case ExAws.request(op, ex_aws_config(config)) do {:ok, _} -> :ok {:error, r} -> {:error, r} end end
defp abort_multipart(config, key, upload_id) do op = ExAws.S3.abort_multipart_upload(config.bucket, key, upload_id) _ = ExAws.request(op, ex_aws_config(config)) :ok end
defp ex_aws_config(%{ex_aws_config: cfg}), do: cfg defp ex_aws_config(_), do: []
defp ex_aws_config_struct(config) do ExAws.Config.new(:s3, ex_aws_config(config)) end end
|
|
|
lib/ex_git_objectstore/lfs/transfer.ex
|
+118
−0
|
@@ -1,0 +1,118 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Transfer do @moduledoc """ Basic-transfer handler for the Git LFS API.
Called by the consumer's HTTP layer after the batch API has returned library-served URLs:
* `GET /objects/:oid` → `download/2` * `PUT /objects/:oid` → `upload/3` * `POST /objects/:oid/verify` → `verify/3`
Each function returns pure data; the consumer converts it to HTTP status codes and headers. """
alias ExGitObjectstore.Lfs.Store alias ExGitObjectstore.Repo
@type result_ok :: %{size: non_neg_integer(), stream: Enumerable.t()}
@doc """ Download an object. Returns a size + chunk stream, or a reason:
* `:lfs_not_configured` * `:bad_oid` * `:not_found` """ @spec download(Repo.t(), String.t()) :: {:ok, result_ok()} | {:error, atom()} def download(%Repo{} = repo, oid) do :telemetry.span( [:ex_git_objectstore, :lfs, :transfer], %{operation: :download, oid: oid, repo: repo.id}, fn -> result = do_download(repo, oid) {result, %{operation: :download, oid: oid, outcome: outcome(result)}} end ) end
defp do_download(%Repo{lfs_storage: nil}, _oid), do: {:error, :lfs_not_configured}
defp do_download(repo, oid) do with :ok <- Store.validate_oid(oid) do Store.call(repo, :get, [oid]) end end
@doc """ Upload an object. Body is a chunk-producing enumerable. The backend verifies the observed SHA256 matches the claimed `oid` and rejects with `:oid_mismatch` on failure. Returns `{:ok, bytes_written}`. """ @spec upload(Repo.t(), String.t(), Enumerable.t()) :: {:ok, non_neg_integer()} | {:error, atom()} def upload(%Repo{} = repo, oid, stream) do :telemetry.span( [:ex_git_objectstore, :lfs, :transfer], %{operation: :upload, oid: oid, repo: repo.id}, fn -> result = do_upload(repo, oid, stream) {result, %{operation: :upload, oid: oid, outcome: outcome(result)}} end ) end
defp do_upload(%Repo{lfs_storage: nil}, _oid, _stream), do: {:error, :lfs_not_configured}
defp do_upload(repo, oid, stream) do with :ok <- Store.validate_oid(oid) do Store.call(repo, :put, [oid, stream]) end end
@doc """ Verify that `oid` exists and its stored size matches `size`. Called by clients after uploading via a presigned URL. """ @spec verify(Repo.t(), String.t(), non_neg_integer()) :: :ok | {:error, atom()} def verify(%Repo{} = repo, oid, size) when is_integer(size) and size >= 0 do :telemetry.span( [:ex_git_objectstore, :lfs, :transfer], %{operation: :verify, oid: oid, repo: repo.id}, fn -> result = do_verify(repo, oid, size) {result, %{operation: :verify, oid: oid, outcome: outcome(result)}} end ) end
def verify(_repo, _oid, _size), do: {:error, :bad_size}
defp do_verify(%Repo{lfs_storage: nil}, _oid, _size), do: {:error, :lfs_not_configured}
defp do_verify(repo, oid, size) do with :ok <- Store.validate_oid(oid), {:ok, %{size: actual}} <- Store.call(repo, :stat, [oid]) do if actual == size, do: :ok, else: {:error, :size_mismatch} end end
defp outcome(:ok), do: :ok defp outcome({:ok, _}), do: :ok defp outcome({:error, reason}), do: reason end
|
|
|
lib/ex_git_objectstore/repo.ex
|
+5
−1
|
@@ -24,11 +24,12 @@ id: String.t(), storage: {module(), map()}, cache: {module(), map()} | nil, lfs_storage: {module(), map()} | nil, max_object_size: pos_integer() }
@enforce_keys [:id, :storage] defstruct [:id, :storage, :cache, :lfs_storage, max_object_size: @default_max_object_size] defstruct [:id, :storage, :cache, max_object_size: @default_max_object_size]
@doc """ Create a new repo handle. @@ -37,18 +38,21 @@
* `:storage` - `{module, config}` tuple for the storage backend (required) * `:cache` - `{module, config}` tuple for the cache backend (optional) * `:lfs_storage` - `{module, config}` tuple for the LFS store backend (optional) """ @spec new(String.t(), keyword()) :: t() def new(id, opts \\ []) do validate_repo_id!(id) storage = Keyword.fetch!(opts, :storage) cache = Keyword.get(opts, :cache) lfs_storage = Keyword.get(opts, :lfs_storage) max_object_size = Keyword.get(opts, :max_object_size, @default_max_object_size)
%__MODULE__{ id: id, storage: storage, cache: cache, lfs_storage: lfs_storage, max_object_size: max_object_size } end
|
|
|
mix.exs
|
+3
−1
|
@@ -99,7 +99,9 @@ {:telemetry, "~> 1.0"}, {:ex_doc, "~> 0.34", only: :dev, runtime: false}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false} {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:plug, "~> 1.16", only: :test}, {:bandit, "~> 1.5", only: :test} ] end end
|
|
|
mix.lock
|
+6
−0
|
@@ -1,4 +1,5 @@ %{ "bandit": {:hex, :bandit, "1.10.4", "02b9734c67c5916a008e7eb7e2ba68aaea6f8177094a5f8d95f1fb99069aac17", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "a5faf501042ac1f31d736d9d4a813b3db4ef812e634583b6a457b0928798a51d"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "credo": {:hex, :credo, "1.7.16", "a9f1389d13d19c631cb123c77a813dbf16449a2aebf602f590defa08953309d4", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0562af33756b21f248f066a9119e3890722031b6d199f22e3cf95550e4f1579"}, @@ -10,6 +11,7 @@ "ex_doc": {:hex, :ex_doc, "0.40.1", "67542e4b6dde74811cfd580e2c0149b78010fd13001fda7cfeb2b2c2ffb1344d", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "bcef0e2d360d93ac19f01a85d58f91752d930c0a30e2681145feea6bd3516e00"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, @@ -20,8 +22,12 @@ "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, }
|
|
|
test/ex_git_objectstore/lfs/batch_test.exs
|
+174
−0
|
@@ -1,0 +1,174 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.BatchTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-003", "REQ-LFS-006"]
alias ExGitObjectstore.Lfs.{Batch, Store} alias ExGitObjectstore.Lfs.Store.Memory, as: LfsMemory alias ExGitObjectstore.Repo alias ExGitObjectstore.Storage.Memory
@base_url "https://git.example.com/r/info/lfs"
setup do {:ok, git_pid} = Memory.start_link() {:ok, lfs_pid} = LfsMemory.start_link()
repo = Repo.new("r", storage: {Memory, Memory.config(git_pid)}, lfs_storage: {LfsMemory, LfsMemory.config(lfs_pid)} )
%{repo: repo} end
defp sha256_of(bin), do: :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower)
describe "upload operation" do test "returns upload + verify actions for new object", %{repo: repo} do oid = String.duplicate("a", 64)
request = %{ "operation" => "upload", "objects" => [%{"oid" => oid, "size" => 100}] }
response = Batch.handle(repo, request, base_url: @base_url)
assert response.status == 200 assert response.body["transfer"] == "basic" assert [obj] = response.body["objects"] assert obj["oid"] == oid assert obj["actions"]["upload"]["href"] =~ "/objects/#{oid}" assert obj["actions"]["verify"]["href"] =~ "/verify" end
test "omits actions for object already present", %{repo: repo} do data = "already here" oid = sha256_of(data)
{:ok, _} = Store.call(repo, :put, [oid, [data]])
request = %{ "operation" => "upload", "objects" => [%{"oid" => oid, "size" => byte_size(data)}] }
response = Batch.handle(repo, request, base_url: @base_url) [obj] = response.body["objects"] refute Map.has_key?(obj, "actions") assert obj["authenticated"] == true end end
describe "download operation" do test "returns download action for existing object", %{repo: repo} do data = "download me" oid = sha256_of(data) {:ok, _} = Store.call(repo, :put, [oid, [data]])
request = %{ "operation" => "download", "objects" => [%{"oid" => oid, "size" => byte_size(data)}] }
response = Batch.handle(repo, request, base_url: @base_url) [obj] = response.body["objects"] assert obj["actions"]["download"]["href"] =~ "/objects/#{oid}" end
test "returns 404 error for missing object", %{repo: repo} do missing = String.duplicate("c", 64)
request = %{ "operation" => "download", "objects" => [%{"oid" => missing, "size" => 10}] }
response = Batch.handle(repo, request, base_url: @base_url) [obj] = response.body["objects"] assert obj["error"]["code"] == 404 end
test "returns 422 on size mismatch", %{repo: repo} do data = "small" oid = sha256_of(data) {:ok, _} = Store.call(repo, :put, [oid, [data]])
request = %{ "operation" => "download", "objects" => [%{"oid" => oid, "size" => 999}] }
response = Batch.handle(repo, request, base_url: @base_url) [obj] = response.body["objects"] assert obj["error"]["code"] == 422 end end
describe "validation" do test "rejects missing operation", %{repo: repo} do assert %{status: 422} = Batch.handle(repo, %{"objects" => []}, base_url: @base_url) end
test "rejects invalid operation", %{repo: repo} do req = %{ "operation" => "wat", "objects" => [%{"oid" => String.duplicate("a", 64), "size" => 1}] }
assert %{status: 422} = Batch.handle(repo, req, base_url: @base_url) end
test "rejects bad oid", %{repo: repo} do req = %{"operation" => "upload", "objects" => [%{"oid" => "nope", "size" => 1}]} assert %{status: 422} = Batch.handle(repo, req, base_url: @base_url) end
test "rejects negative size", %{repo: repo} do req = %{ "operation" => "upload", "objects" => [%{"oid" => String.duplicate("a", 64), "size" => -1}] }
assert %{status: 422} = Batch.handle(repo, req, base_url: @base_url) end
test "returns 501 when lfs not configured" do {:ok, git_pid} = Memory.start_link() repo = Repo.new("r", storage: {Memory, Memory.config(git_pid)})
req = %{ "operation" => "upload", "objects" => [%{"oid" => String.duplicate("a", 64), "size" => 1}] }
assert %{status: 501} = Batch.handle(repo, req, base_url: @base_url) end
test "rejects unsupported transfers", %{repo: repo} do req = %{ "operation" => "upload", "transfers" => ["custom-only"], "objects" => [%{"oid" => String.duplicate("a", 64), "size" => 1}] }
assert %{status: 422} = Batch.handle(repo, req, base_url: @base_url) end end end
|
|
|
test/ex_git_objectstore/lfs/interop_s3_test.exs
|
+254
−0
|
@@ -1,0 +1,254 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.InteropS3Test do @moduledoc """ End-to-end interop of the LFS library against the S3 backend (MinIO), driven by the real `git lfs` client.
This exercises a different code path from the Filesystem interop tests: `Batch.handle/3` now emits presigned URLs pointing at MinIO, and `git-lfs` uploads/downloads directly against the object store. The library's HTTP adapter only sees the batch + verify + locks requests.
Skipped unless MinIO is reachable on localhost:9000. """
use ExUnit.Case, async: false
@moduletag :s3 @moduletag requirements: ["REQ-LFS-002", "REQ-LFS-003", "REQ-LFS-004"]
alias ExGitObjectstore.Lfs.Store.S3, as: LfsS3 alias ExGitObjectstore.Repo alias ExGitObjectstore.Storage.Filesystem, as: GitFilesystem alias ExGitObjectstore.Test.LfsHttpAdapter
@minio_config %{ bucket: "test-bucket", ex_aws_config: [ access_key_id: "minioadmin", secret_access_key: "minioadmin", region: "us-east-1", host: "localhost", port: 9000, scheme: "http://", s3_scheme: "http://", s3_host: "localhost", s3_port: 9000 ], multipart_part_size: 5 * 1024 * 1024 }
@minio_available (case :gen_tcp.connect(~c"localhost", 9000, [], 1_000) do {:ok, socket} -> :gen_tcp.close(socket) true
{:error, _} -> false end)
@git_lfs_available (case System.cmd("git", ["lfs", "version"], stderr_to_stdout: true) do {_out, 0} -> true _ -> false end)
if !@minio_available or !@git_lfs_available do @moduletag :skip end
setup_all do if @minio_available, do: ensure_bucket() :ok end
setup do root = tmp!("lfs-s3-interop-root") port = random_port()
factory = fn repo_id -> Repo.new(repo_id, storage: {GitFilesystem, %{root: root}}, lfs_storage: {LfsS3, @minio_config} ) end
{:ok, server} = Bandit.start_link( plug: {LfsHttpAdapter, repo_factory: factory}, scheme: :http, port: port, ip: {127, 0, 0, 1} )
on_exit(fn -> if Process.alive?(server) do try do GenServer.stop(server, :normal, 5_000) catch :exit, _ -> :ok end end end)
%{ root: root, port: port, lfs_url: fn repo_id -> "http://127.0.0.1:#{port}/#{repo_id}/info/lfs" end } end
describe "git lfs push/pull against S3 backend" do @tag timeout: :timer.minutes(2) test "presigned URL round-trip — client uploads to MinIO, downloads from MinIO", ctx do repo_id = "s3_e2e_#{:erlang.unique_integer([:positive])}"
# Publisher: commit an LFS blob and push it. publisher = tmp!("lfs-s3-pub") payload = :crypto.strong_rand_bytes(1 * 1024 * 1024) oid = :crypto.hash(:sha256, payload) |> Base.encode16(case: :lower)
git!(publisher, ["init", "-q", "-b", "main"]) git!(publisher, ["config", "user.email", "test@example.com"]) git!(publisher, ["config", "user.name", "Test"]) git!(publisher, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(publisher, ["lfs", "track", "*.bin"]) File.write!(Path.join(publisher, "big.bin"), payload) git!(publisher, ["add", "."]) git!(publisher, ["commit", "-qm", "seed"])
# Push to the LFS server. Batch returns a presigned PUT URL for MinIO; # git-lfs uploads directly to MinIO; git-lfs then POSTs /verify. git!(publisher, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
# Verify the object is now in MinIO via the library's Store API. repo = Repo.new(repo_id, storage: {GitFilesystem, %{root: ctx.root}}, lfs_storage: {LfsS3, @minio_config} )
assert LfsS3.exists?(@minio_config, "repos/#{repo_id}/lfs", oid) assert {:ok, %{size: size}} = LfsS3.stat(@minio_config, "repos/#{repo_id}/lfs", oid) assert size == byte_size(payload)
# Read bytes back through the backend directly — sanity check that the # presigned-upload actually deposited the correct content. {:ok, %{stream: stream}} = LfsS3.get(@minio_config, "repos/#{repo_id}/lfs", oid) fetched = stream |> Enum.to_list() |> IO.iodata_to_binary() assert fetched == payload
# Download path: drive git-lfs smudge against our server, which will # receive a presigned download URL and pull the bytes from MinIO. fresh = tmp!("lfs-s3-fresh")
pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:#{oid}\nsize #{byte_size(payload)}\n"
git!(fresh, ["init", "-q", "-b", "main"]) git!(fresh, ["config", "user.email", "test@example.com"]) git!(fresh, ["config", "user.name", "Test"]) git!(fresh, ["config", "lfs.url", ctx.lfs_url.(repo_id)])
File.write!( Path.join(fresh, ".gitattributes"), "*.bin filter=lfs diff=lfs merge=lfs -text\n" )
smudged = run_smudge(fresh, pointer) assert :crypto.hash(:sha256, smudged) == :crypto.hash(:sha256, payload)
_ = repo end
test "multi-object push lands all objects on S3", ctx do repo_id = "s3_multi_#{:erlang.unique_integer([:positive])}" work = tmp!("lfs-s3-multi")
git!(work, ["init", "-q", "-b", "main"]) git!(work, ["config", "user.email", "test@example.com"]) git!(work, ["config", "user.name", "Test"]) git!(work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(work, ["lfs", "track", "*.bin"])
payloads = for i <- 1..4 do bytes = :crypto.strong_rand_bytes(128 * 1024 + i * 1024) File.write!(Path.join(work, "f#{i}.bin"), bytes) bytes end
git!(work, ["add", "."]) git!(work, ["commit", "-qm", "multi"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
for bytes <- payloads do oid = :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower)
assert LfsS3.exists?(@minio_config, "repos/#{repo_id}/lfs", oid), "object #{oid} should be in S3" end end end
# -- helpers --
defp ensure_bucket do op = ExAws.S3.put_bucket(@minio_config.bucket, @minio_config.ex_aws_config[:region]) _ = ExAws.request(op, @minio_config.ex_aws_config) :ok end
defp tmp!(label) do dir = Path.join(System.tmp_dir!(), "#{label}-#{:erlang.unique_integer([:positive])}") File.mkdir_p!(dir) on_exit(fn -> File.rm_rf!(dir) end) dir end
defp random_port do {:ok, sock} = :gen_tcp.listen(0, [:binary, {:ip, {127, 0, 0, 1}}]) {:ok, port} = :inet.port(sock) :gen_tcp.close(sock) port end
defp git!(dir, args) do {output, status} = System.cmd("git", args, cd: dir, stderr_to_stdout: true)
if status != 0 do raise "git #{Enum.join(args, " ")} failed (#{status}): #{output}" end
String.trim(output) end
defp run_smudge(cwd, pointer) do pointer_file = Path.join(cwd, ".pointer.in") output_file = Path.join(cwd, ".smudge.out") File.write!(pointer_file, pointer)
script = "git lfs smudge big.bin < #{Path.basename(pointer_file)} > #{Path.basename(output_file)}"
{stderr_out, status} = System.cmd("sh", ["-c", script], cd: cwd, stderr_to_stdout: true)
if status != 0 do raise "git lfs smudge exited #{status}: #{stderr_out}" end
File.read!(output_file) end end
|
|
|
test/ex_git_objectstore/lfs/interop_test.exs
|
+575
−0
|
@@ -1,0 +1,575 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.InteropTest do @moduledoc """ End-to-end interop against the real `git lfs` binary.
Runs a Bandit server that wires the library's LFS modules to HTTP, then drives the real `git lfs` client through push / pull / lock flows. This file is skipped automatically when `git-lfs` is not on PATH. """
use ExUnit.Case, async: false
@moduletag requirements: [ "REQ-LFS-001", "REQ-LFS-002", "REQ-LFS-003", "REQ-LFS-004", "REQ-LFS-005" ]
alias ExGitObjectstore.Lfs.Store.Filesystem, as: LfsFilesystem alias ExGitObjectstore.Repo alias ExGitObjectstore.Storage.Filesystem, as: GitFilesystem alias ExGitObjectstore.Test.LfsHttpAdapter
@git_lfs_available (case System.cmd("git", ["lfs", "version"], stderr_to_stdout: true) do {_out, 0} -> true _ -> false end)
if !@git_lfs_available do @moduletag :skip end
setup do root = tmp!("lfs-interop-root") port = random_port()
factory = fn repo_id -> Repo.new(repo_id, storage: {GitFilesystem, %{root: root}}, lfs_storage: {LfsFilesystem, %{root: root}} ) end
{:ok, server} = Bandit.start_link( plug: {LfsHttpAdapter, repo_factory: factory}, scheme: :http, port: port, ip: {127, 0, 0, 1} )
on_exit(fn -> if Process.alive?(server) do ref = Process.monitor(server)
try do GenServer.stop(server, :normal, 5_000) catch :exit, _ -> :ok end
receive do {:DOWN, ^ref, _, _, _} -> :ok after 5_000 -> :ok end end end)
%{ root: root, port: port, lfs_url: fn repo_id -> "http://127.0.0.1:#{port}/#{repo_id}/info/lfs" end, git_url: fn repo_id -> "http://127.0.0.1:#{port}/#{repo_id}" end } end
describe "git lfs push/pull" do test "round-trips a large binary file via the basic transfer adapter", ctx do repo_id = "interop_#{:erlang.unique_integer([:positive])}" work = tmp!("lfs-work") payload = :crypto.strong_rand_bytes(3 * 1024 * 1024)
# 1. Initialise a local working copy, set up git-lfs git!(work, ["init", "-q", "-b", "main"]) git!(work, ["config", "user.email", "test@example.com"]) git!(work, ["config", "user.name", "Test"]) git!(work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(work, ["lfs", "track", "*.bin"]) File.write!(Path.join(work, "big.bin"), payload) git!(work, ["add", ".gitattributes", "big.bin"]) git!(work, ["commit", "-qm", "add lfs blob"])
# 2. Push the LFS object (the pointer blob + the big blob content) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
# 3. Verify the object now lives on the server filesystem oid = :crypto.hash(:sha256, payload) |> Base.encode16(case: :lower)
server_path = Path.join([ ctx.root, "repos", repo_id, "lfs", String.slice(oid, 0, 2), String.slice(oid, 2, 2), oid ])
assert File.exists?(server_path), "server should have stored the LFS blob at #{server_path}" assert File.read!(server_path) == payload
# 4. Exercise the download path — drive git-lfs as a client fetching the # object from our server. Uses `git lfs fetch` which resolves the OID # via the batch API and GETs the object. fresh = tmp!("lfs-fresh")
pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:#{oid}\nsize #{byte_size(payload)}\n"
git!(fresh, ["init", "-q", "-b", "main"]) git!(fresh, ["config", "user.email", "test@example.com"]) git!(fresh, ["config", "user.name", "Test"]) git!(fresh, ["config", "lfs.url", ctx.lfs_url.(repo_id)])
File.write!( Path.join(fresh, ".gitattributes"), "*.bin filter=lfs diff=lfs merge=lfs -text\n" )
# Smudge reads a pointer from stdin, asks the batch API for a download # action for that OID, GETs the bytes, and writes them to stdout. Drive # it via an Erlang Port so we can send the pointer on stdin. smudged = run_smudge(fresh, pointer)
assert byte_size(smudged) == byte_size(payload) assert :crypto.hash(:sha256, smudged) == :crypto.hash(:sha256, payload) end end
defp run_smudge(cwd, pointer) do pointer_file = Path.join(cwd, ".pointer.in") output_file = Path.join(cwd, ".smudge.out") File.write!(pointer_file, pointer)
# System.cmd doesn't support stdin redirection, so shell it out. script = "git lfs smudge big.bin < #{Path.basename(pointer_file)} > #{Path.basename(output_file)}"
{stderr_out, status} = System.cmd("sh", ["-c", script], cd: cwd, stderr_to_stdout: true)
if status != 0 do raise "git lfs smudge exited #{status}: #{stderr_out}" end
File.read!(output_file) end
describe "git lfs push (idempotency)" do test "pushing the same object twice is a no-op the second time", ctx do repo_id = "interop_idem_#{:erlang.unique_integer([:positive])}" work = tmp!("lfs-idem") payload = :crypto.strong_rand_bytes(256 * 1024)
git!(work, ["init", "-q", "-b", "main"]) git!(work, ["config", "user.email", "test@example.com"]) git!(work, ["config", "user.name", "Test"]) git!(work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(work, ["lfs", "track", "*.bin"]) File.write!(Path.join(work, "x.bin"), payload) git!(work, ["add", ".gitattributes", "x.bin"]) git!(work, ["commit", "-qm", "initial"])
first = git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)]) assert is_binary(first)
# Second push: the object already exists server-side, so the batch # response should omit the upload action and the client should skip # the transfer entirely. We assert the exit code is still 0. second = git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)]) assert is_binary(second) end end
describe "git lfs lock/unlock" do test "round-trips lock creation and release", ctx do repo_id = "interop_locks_#{:erlang.unique_integer([:positive])}" work = tmp!("lfs-lock-work")
git!(work, ["init", "-q", "-b", "main"]) git!(work, ["config", "user.email", "alice@example.com"]) git!(work, ["config", "user.name", "alice"]) git!(work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(work, ["lfs", "track", "*.psd"]) File.write!(Path.join(work, "design.psd"), "placeholder") git!(work, ["add", ".gitattributes", "design.psd"]) git!(work, ["commit", "-qm", "track"])
# Acquire a lock lock_out = git!(work, ["lfs", "lock", "design.psd"]) assert lock_out =~ "Locked design.psd" or lock_out =~ "design.psd"
# List locks — should see our entry list_out = git!(work, ["lfs", "locks"]) assert list_out =~ "design.psd"
# A second lock on the same path must conflict. {dup_out, dup_status} = System.cmd("git", ["lfs", "lock", "design.psd"], cd: work, stderr_to_stdout: true )
assert dup_status != 0, "duplicate lock should fail" assert dup_out =~ "already" or dup_out =~ "conflict" or dup_out =~ "locked"
# Release it unlock_out = git!(work, ["lfs", "unlock", "design.psd"]) assert unlock_out =~ "Unlocked design.psd" or unlock_out =~ "design.psd"
# List again — empty list_out2 = git!(work, ["lfs", "locks"]) refute list_out2 =~ "design.psd" end end
describe "git lfs push — edge-case payload sizes" do test "empty file upload via direct HTTP (git-lfs client elides this)", ctx do # The git-lfs client does not actually upload a 0-byte object — it # notices the well-known empty-sha256 and skips the transfer. Exercise # the server path directly to prove the library accepts it. repo_id = "interop_empty_#{:erlang.unique_integer([:positive])}" oid = sha256_hex("")
url = ctx.lfs_url.(repo_id) <> "/objects/" <> oid {:ok, {{_, status, _}, _headers, _body}} = http_put(url, "")
assert status == 200 assert server_object_bytes(ctx.root, repo_id, oid) == "" end
test "single-byte file", ctx do repo_id = "interop_one_#{:erlang.unique_integer([:positive])}" work = setup_client_repo(ctx, repo_id, "lfs-one", "*.bin") File.write!(Path.join(work, "one.bin"), "x") git!(work, ["add", "one.bin"]) git!(work, ["commit", "-qm", "one"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
oid = sha256_hex("x") assert server_object_bytes(ctx.root, repo_id, oid) == "x" end end
describe "git lfs push — multiple objects" do test "pushes N distinct files in one batch", ctx do repo_id = "interop_multi_#{:erlang.unique_integer([:positive])}" work = setup_client_repo(ctx, repo_id, "lfs-multi", "*.bin")
files = for i <- 1..5 do name = "f#{i}.bin" bytes = :crypto.strong_rand_bytes(64 * 1024 * i) File.write!(Path.join(work, name), bytes) {name, bytes} end
git!(work, ["add", "."]) git!(work, ["commit", "-qm", "multi"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
for {_name, bytes} <- files do oid = sha256_hex(bytes) assert server_object_bytes(ctx.root, repo_id, oid) == bytes end end
test "mixed-state batch skips present objects and uploads new ones", ctx do repo_id = "interop_mixed_#{:erlang.unique_integer([:positive])}" work = setup_client_repo(ctx, repo_id, "lfs-mixed", "*.bin")
a = :crypto.strong_rand_bytes(32 * 1024) b = :crypto.strong_rand_bytes(32 * 1024)
File.write!(Path.join(work, "a.bin"), a) git!(work, ["add", "."]) git!(work, ["commit", "-qm", "first"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
File.write!(Path.join(work, "b.bin"), b) git!(work, ["add", "b.bin"]) git!(work, ["commit", "-qm", "second"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
assert server_object_bytes(ctx.root, repo_id, sha256_hex(a)) == a assert server_object_bytes(ctx.root, repo_id, sha256_hex(b)) == b end end
describe "git lfs push — concurrent transfers" do test "git-lfs parallel-upload path lands every object without duplication", ctx do repo_id = "interop_par_#{:erlang.unique_integer([:positive])}" work = setup_client_repo(ctx, repo_id, "lfs-par", "*.bin")
payloads = for i <- 1..10 do name = "p#{i}.bin" bytes = :crypto.strong_rand_bytes(128 * 1024 + i) File.write!(Path.join(work, name), bytes) {name, bytes} end
git!(work, ["add", "."]) git!(work, ["commit", "-qm", "parallel"])
# git-lfs default transfer concurrency is 3; force it higher to stress # the server with overlapping PUTs. git!(work, ["config", "lfs.concurrenttransfers", "8"]) git!(work, ["lfs", "push", "--all", ctx.lfs_url.(repo_id)])
for {_name, bytes} <- payloads do oid = sha256_hex(bytes) assert server_object_bytes(ctx.root, repo_id, oid) == bytes end end end
describe "upload OID tampering (direct HTTP)" do test "server rejects body whose sha256 doesn't match the URL's oid", ctx do repo_id = "interop_tamper_#{:erlang.unique_integer([:positive])}" # OID claims to be sha256 of "hello" but we send "goodbye" claimed = sha256_hex("hello") body = "goodbye"
url = ctx.lfs_url.(repo_id) <> "/objects/" <> claimed {:ok, {{_, status, _}, _headers, resp_body}} = http_put(url, body)
assert status == 422 assert resp_body =~ "mismatch" or resp_body =~ "sha256"
refute server_object_exists?(ctx.root, repo_id, claimed), "server must not persist a mismatched upload" end end
describe "git lfs locks — verify and force-unlock" do test "locks verify splits ours vs theirs and force unlocks foreign lock", ctx do repo_id = "interop_lock_force_#{:erlang.unique_integer([:positive])}"
alice_work = tmp!("lfs-alice") bob_work = tmp!("lfs-bob")
# Alice locks a.psd git!(alice_work, ["init", "-q", "-b", "main"]) git!(alice_work, ["config", "user.email", "alice@example.com"]) git!(alice_work, ["config", "user.name", "alice"]) git!(alice_work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(alice_work, ["lfs", "track", "*.psd"]) File.write!(Path.join(alice_work, "a.psd"), "a") git!(alice_work, ["add", "."]) git!(alice_work, ["commit", "-qm", "a"]) git!(alice_work, ["lfs", "lock", "a.psd"])
# Bob locks b.psd git!(bob_work, ["init", "-q", "-b", "main"]) git!(bob_work, ["config", "user.email", "bob@example.com"]) git!(bob_work, ["config", "user.name", "bob"]) git!(bob_work, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(bob_work, ["lfs", "track", "*.psd"]) File.write!(Path.join(bob_work, "b.psd"), "b") git!(bob_work, ["add", "."]) git!(bob_work, ["commit", "-qm", "b"]) git!(bob_work, ["lfs", "lock", "b.psd"])
# From Alice's POV: ours=[a.psd] theirs=[b.psd]. git-lfs `locks` with # --verify flag goes through /locks/verify rather than /locks. verify_out = git!(alice_work, ["lfs", "locks", "--verify"]) assert verify_out =~ "a.psd" assert verify_out =~ "b.psd"
# Find bob's lock id via the locks list locks_raw = git!(alice_work, ["lfs", "locks", "--json"]) b_lock = Enum.find(Jason.decode!(locks_raw), fn l -> l["path"] == "b.psd" end) refute is_nil(b_lock) b_id = b_lock["id"]
# Server must 403 when a non-owner tries to unlock without force. # (The git-lfs client short-circuits client-side — refusing to even # send the request — so exercise the path directly.) url = ctx.lfs_url.(repo_id) <> "/locks/" <> b_id <> "/unlock"
{:ok, {{_, status, _}, _, _}} = http_post_json(url, %{"force" => false}, [{~c"x-lfs-user", ~c"alice"}])
assert status == 403
# With --force the CLI will send it forced = git!(alice_work, ["lfs", "unlock", "--force", "b.psd"]) assert forced =~ "b.psd"
# Locks list should be down to just a.psd remaining = git!(alice_work, ["lfs", "locks"]) assert remaining =~ "a.psd" refute remaining =~ "b.psd" end end
describe "full end-to-end: git push + git clone + git lfs pull" do @tag timeout: :timer.minutes(2) test "clone-then-pull round-trips both git commits and LFS content", ctx do repo_id = "interop_e2e_#{:erlang.unique_integer([:positive])}"
# -------- seed the server by pushing from a publisher repo -------- publisher = setup_client_repo(ctx, repo_id, "lfs-e2e-publish", "*.bin")
# .lfsconfig makes the LFS URL discoverable after clone File.write!( Path.join(publisher, ".lfsconfig"), "[lfs]\n\turl = #{ctx.lfs_url.(repo_id)}\n" )
payload = :crypto.strong_rand_bytes(512 * 1024) oid = sha256_hex(payload) File.write!(Path.join(publisher, "big.bin"), payload) File.write!(Path.join(publisher, "readme.txt"), "hello world\n")
git!(publisher, ["add", "."]) git!(publisher, ["commit", "-qm", "seed"])
# Explicit remote pointing at our smart-http endpoint git!(publisher, ["remote", "add", "origin", ctx.git_url.(repo_id)])
# Push git refs first (our receive-pack), then LFS content git!(publisher, ["push", "-u", "origin", "main"]) git!(publisher, ["lfs", "push", "--all", "origin"])
# Sanity: the blob object landed on the server assert server_object_bytes(ctx.root, repo_id, oid) == payload
# -------- fresh client: git clone then git lfs pull -------- clone_parent = tmp!("lfs-e2e-clone-parent")
# git clone <url> <dir> {out, status} = System.cmd("git", ["clone", "-q", ctx.git_url.(repo_id), "cloned"], cd: clone_parent, stderr_to_stdout: true, env: [{"GIT_LFS_SKIP_SMUDGE", "1"}] )
assert status == 0, "git clone failed: #{out}"
cloned = Path.join(clone_parent, "cloned")
# Regular file came through the pack assert File.read!(Path.join(cloned, "readme.txt")) == "hello world\n"
# big.bin is a pointer until we pull LFS content pointer_contents = File.read!(Path.join(cloned, "big.bin")) assert pointer_contents =~ "version https://git-lfs.github.com/spec/v1" assert pointer_contents =~ "oid sha256:#{oid}"
# Pull LFS content — exercises batch (download) + object GET git!(cloned, ["lfs", "pull"])
# big.bin is now the real content assert File.read!(Path.join(cloned, "big.bin")) == payload end end
# -- Helpers --
defp setup_client_repo(ctx, repo_id, label, attr_pattern) do dir = tmp!(label) git!(dir, ["init", "-q", "-b", "main"]) git!(dir, ["config", "user.email", "test@example.com"]) git!(dir, ["config", "user.name", "Test"]) git!(dir, ["config", "lfs.url", ctx.lfs_url.(repo_id)]) git!(dir, ["lfs", "track", attr_pattern]) git!(dir, ["add", ".gitattributes"]) dir end
defp sha256_hex(bin), do: :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower)
defp server_object_path(root, repo_id, oid) do Path.join([ root, "repos", repo_id, "lfs", String.slice(oid, 0, 2), String.slice(oid, 2, 2), oid ]) end
defp server_object_exists?(root, repo_id, oid), do: File.exists?(server_object_path(root, repo_id, oid))
defp server_object_bytes(root, repo_id, oid), do: File.read!(server_object_path(root, repo_id, oid))
defp http_put(url, body) do :inets.start() :ssl.start()
headers = [{~c"content-length", ~c"#{byte_size(body)}"}] request = {String.to_charlist(url), headers, ~c"application/octet-stream", body}
:httpc.request(:put, request, [], body_format: :binary) end
defp http_post_json(url, body, extra_headers) do :inets.start() :ssl.start()
json = Jason.encode!(body) headers = extra_headers ++ [{~c"content-length", ~c"#{byte_size(json)}"}] request = {String.to_charlist(url), headers, ~c"application/vnd.git-lfs+json", json}
:httpc.request(:post, request, [], body_format: :binary) end
defp tmp!(label) do dir = Path.join(System.tmp_dir!(), "#{label}-#{:erlang.unique_integer([:positive])}") File.mkdir_p!(dir) on_exit(fn -> File.rm_rf!(dir) end) dir end
defp random_port do {:ok, sock} = :gen_tcp.listen(0, [:binary, {:ip, {127, 0, 0, 1}}]) {:ok, port} = :inet.port(sock) :gen_tcp.close(sock) port end
defp git!(dir, args) do {output, status} = System.cmd("git", args, cd: dir, stderr_to_stdout: true, env: [ # Make lock requests predictable in tests — lfs uses the email as the owner name # unless we override via protocol extension headers. {"GIT_LFS_SKIP_SMUDGE", "1"} ] )
if status != 0 do raise "git #{Enum.join(args, " ")} failed (#{status}): #{output}" end
String.trim(output) end end
|
|
|
test/ex_git_objectstore/lfs/locks_test.exs
|
+147
−0
|
@@ -1,0 +1,147 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.LocksTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-005", "REQ-LFS-006"]
alias ExGitObjectstore.Lfs.Locks alias ExGitObjectstore.Repo alias ExGitObjectstore.Storage.Memory
setup do {:ok, pid} = Memory.start_link() repo = Repo.new("r", storage: {Memory, Memory.config(pid)}) %{repo: repo} end
describe "create/3" do test "creates a lock on a new path", %{repo: repo} do assert {:ok, lock} = Locks.create(repo, "big/file.psd", "alice") assert lock.path == "big/file.psd" assert lock.owner.name == "alice" assert is_binary(lock.id) and byte_size(lock.id) == 32 assert is_binary(lock.locked_at) end
test "rejects a second lock on the same path", %{repo: repo} do {:ok, first} = Locks.create(repo, "p", "alice") assert {:error, {:conflict, ^first}} = Locks.create(repo, "p", "bob") end
test "rejects empty path", %{repo: repo} do assert {:error, :bad_request} = Locks.create(repo, "", "alice") end end
describe "list/2" do test "returns all locks", %{repo: repo} do {:ok, a} = Locks.create(repo, "a", "alice") {:ok, b} = Locks.create(repo, "b", "bob")
{:ok, locks} = Locks.list(repo) assert length(locks) == 2 assert Enum.map(locks, & &1.path) |> Enum.sort() == [a.path, b.path] end
test "filters by path", %{repo: repo} do {:ok, _} = Locks.create(repo, "a", "alice") {:ok, _} = Locks.create(repo, "b", "bob") {:ok, [lock]} = Locks.list(repo, path: "a") assert lock.path == "a" end
test "filters by id", %{repo: repo} do {:ok, a} = Locks.create(repo, "a", "alice") {:ok, _} = Locks.create(repo, "b", "bob") {:ok, [lock]} = Locks.list(repo, id: a.id) assert lock.id == a.id end
test "returns empty when no locks", %{repo: repo} do assert {:ok, []} = Locks.list(repo) end end
describe "verify/2" do test "splits locks by owner", %{repo: repo} do {:ok, _} = Locks.create(repo, "a", "alice") {:ok, _} = Locks.create(repo, "b", "bob") {:ok, _} = Locks.create(repo, "c", "alice")
{:ok, %{ours: ours, theirs: theirs}} = Locks.verify(repo, "alice") assert Enum.map(ours, & &1.path) |> Enum.sort() == ["a", "c"] assert Enum.map(theirs, & &1.path) == ["b"] end end
describe "unlock/4" do test "owner can unlock their own lock", %{repo: repo} do {:ok, lock} = Locks.create(repo, "p", "alice") assert {:ok, ^lock} = Locks.unlock(repo, lock.id, "alice") assert {:ok, []} = Locks.list(repo) end
test "non-owner cannot unlock without force", %{repo: repo} do {:ok, lock} = Locks.create(repo, "p", "alice") assert {:error, :forbidden} = Locks.unlock(repo, lock.id, "bob") assert {:ok, [_]} = Locks.list(repo) end
test "non-owner with force can unlock", %{repo: repo} do {:ok, lock} = Locks.create(repo, "p", "alice") assert {:ok, ^lock} = Locks.unlock(repo, lock.id, "bob", force: true) assert {:ok, []} = Locks.list(repo) end
test "returns :not_found for missing id", %{repo: repo} do assert {:error, :not_found} = Locks.unlock(repo, "deadbeef", "alice") end end
describe "telemetry" do test "emits span events for create and unlock", %{repo: repo} do parent = self() ref = make_ref()
:telemetry.attach_many( ref, [ [:ex_git_objectstore, :lfs, :lock, :start], [:ex_git_objectstore, :lfs, :lock, :stop] ], fn event, _m, meta, _ -> send(parent, {:tele, ref, event, meta}) end, nil )
on_exit(fn -> :telemetry.detach(ref) end)
{:ok, lock} = Locks.create(repo, "telemetry-path", "alice") {:ok, ^lock} = Locks.unlock(repo, lock.id, "alice")
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :lock, :start], %{operation: :create}}
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :lock, :stop], %{operation: :create, outcome: :ok}}
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :lock, :start], %{operation: :unlock}}
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :lock, :stop], %{operation: :unlock, outcome: :ok}} end end end
|
|
|
test/ex_git_objectstore/lfs/pointer_test.exs
|
+318
−0
|
@@ -1,0 +1,318 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.PointerTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-001"]
alias ExGitObjectstore.Lfs.Pointer
@oid String.duplicate("a", 64)
describe "parse/1" do test "parses a minimal spec-compliant pointer" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size 12345 """
assert {:ok, ptr} = Pointer.parse(blob) assert ptr.oid == @oid assert ptr.size == 12_345 assert ptr.extra == %{} end
test "parses pointer with extra keys in alphabetical order" do blob = """ version https://git-lfs.github.com/spec/v1 ext.alpha foo ext.beta bar oid sha256:#{@oid} size 42 """
assert {:ok, ptr} = Pointer.parse(blob) assert ptr.oid == @oid assert ptr.size == 42 assert ptr.extra == %{"ext.alpha" => "foo", "ext.beta" => "bar"} end
test "rejects pointer missing trailing LF" do blob = "version https://git-lfs.github.com/spec/v1\noid sha256:#{@oid}\nsize 1"
assert {:error, :missing_trailing_lf} = Pointer.parse(blob) end
test "rejects pointer with wrong version" do blob = """ version https://git-lfs.github.com/spec/v2 oid sha256:#{@oid} size 1 """
assert {:error, :bad_version} = Pointer.parse(blob) end
test "rejects pointer where version is not first" do blob = """ oid sha256:#{@oid} version https://git-lfs.github.com/spec/v1 size 1 """
assert {:error, :version_not_first} = Pointer.parse(blob) end
test "rejects non-sha256 oid" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha1:#{String.duplicate("a", 40)} size 1 """
assert {:error, :bad_oid} = Pointer.parse(blob) end
test "rejects malformed oid hex" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{String.duplicate("Z", 64)} size 1 """
assert {:error, :bad_oid} = Pointer.parse(blob) end
test "rejects oid wrong length" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{String.duplicate("a", 63)} size 1 """
assert {:error, :bad_oid} = Pointer.parse(blob) end
test "rejects non-integer size" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size twelve """
assert {:error, :bad_size} = Pointer.parse(blob) end
test "rejects negative size" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size -1 """
assert {:error, :bad_size} = Pointer.parse(blob) end
test "rejects size with leading plus" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size +1 """
assert {:error, :bad_size} = Pointer.parse(blob) end
test "rejects size with leading zeros (except 0 itself)" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size 007 """
assert {:error, :bad_size} = Pointer.parse(blob) end
test "accepts size of exactly 0" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size 0 """
assert {:ok, ptr} = Pointer.parse(blob) assert ptr.size == 0 end
test "rejects missing oid" do blob = """ version https://git-lfs.github.com/spec/v1 size 1 """
assert {:error, :missing_oid} = Pointer.parse(blob) end
test "rejects missing size" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} """
assert {:error, :missing_size} = Pointer.parse(blob) end
test "rejects keys out of alphabetical order" do blob = """ version https://git-lfs.github.com/spec/v1 size 1 oid sha256:#{@oid} """
assert {:error, :keys_unsorted} = Pointer.parse(blob) end
test "rejects duplicate keys" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} oid sha256:#{String.duplicate("b", 64)} size 1 """
assert {:error, :duplicate_key} = Pointer.parse(blob) end
test "rejects keys with illegal characters" do blob = """ version https://git-lfs.github.com/spec/v1 BADKEY x oid sha256:#{@oid} size 1 """
assert {:error, :bad_key} = Pointer.parse(blob) end
test "rejects malformed line (no space)" do blob = """ version https://git-lfs.github.com/spec/v1 oidsha256:#{@oid} size 1 """
assert {:error, :bad_line} = Pointer.parse(blob) end
test "rejects CRLF line endings" do blob = "version https://git-lfs.github.com/spec/v1\r\noid sha256:#{@oid}\r\nsize 1\r\n"
assert {:error, _} = Pointer.parse(blob) end
test "rejects oversized pointer blob" do huge = String.duplicate("x", 200 * 1024)
blob = """ version https://git-lfs.github.com/spec/v1 ext.big #{huge} oid sha256:#{@oid} size 1 """
assert {:error, :too_large} = Pointer.parse(blob) end
test "pointer?/1 returns true for valid pointer" do blob = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size 1 """
assert Pointer.pointer?(blob) end
test "pointer?/1 returns false for arbitrary binary data" do refute Pointer.pointer?("hello world\n") refute Pointer.pointer?(<<0, 1, 2, 3, 4>>) refute Pointer.pointer?("") end end
describe "emit/1" do test "emits canonical form of parsed pointer" do input = """ version https://git-lfs.github.com/spec/v1 oid sha256:#{@oid} size 42 """
assert {:ok, ptr} = Pointer.parse(input) assert Pointer.emit(ptr) == input end
test "emits extras in alphabetical order" do ptr = %Pointer{ oid: @oid, size: 1, extra: %{"ext.zeta" => "z", "ext.alpha" => "a"} }
out = Pointer.emit(ptr)
assert out == """ version https://git-lfs.github.com/spec/v1 ext.alpha a ext.zeta z oid sha256:#{@oid} size 1 """ end
test "round-trips any valid parsed pointer" do blob = """ version https://git-lfs.github.com/spec/v1 ext.a 1 ext.b 2 oid sha256:#{@oid} size 99 """
assert {:ok, ptr} = Pointer.parse(blob) assert {:ok, ^ptr} = Pointer.parse(Pointer.emit(ptr)) end end
describe "new/2" do test "constructs a pointer from oid and size" do assert {:ok, ptr} = Pointer.new(@oid, 100) assert ptr.oid == @oid assert ptr.size == 100 assert ptr.extra == %{} end
test "rejects bad oid" do assert {:error, :bad_oid} = Pointer.new("short", 1) end
test "rejects negative size" do assert {:error, :bad_size} = Pointer.new(@oid, -1) end end end
|
|
|
test/ex_git_objectstore/lfs/store/filesystem_test.exs
|
+77
−0
|
@@ -1,0 +1,77 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.FilesystemTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-002"]
use ExGitObjectstore.Test.LfsStoreConformance
alias ExGitObjectstore.Lfs.Store.Filesystem
setup do root = Path.join(System.tmp_dir!(), "lfs-fs-#{:erlang.unique_integer([:positive])}") File.mkdir_p!(root) on_exit(fn -> File.rm_rf!(root) end) %{mod: Filesystem, cfg: %{root: root}, prefix: "repos/t/lfs"} end
describe "filesystem-specific behaviour" do setup do root = Path.join(System.tmp_dir!(), "lfs-fs-#{:erlang.unique_integer([:positive])}") File.mkdir_p!(root) on_exit(fn -> File.rm_rf!(root) end) %{cfg: %{root: root}, prefix: "repos/t/lfs"} end
test "path traversal in prefix is rejected", %{cfg: cfg} do assert_raise ArgumentError, fn -> Filesystem.exists?(cfg, "../../../etc", String.duplicate("a", 64)) end end
test "oid mismatch leaves no file on disk", %{cfg: cfg, prefix: pfx} do wrong = String.duplicate("a", 64) assert {:error, :oid_mismatch} = Filesystem.put(cfg, pfx, wrong, ["not-a"]) refute Filesystem.exists?(cfg, pfx, wrong)
# Verify no tmp leaks in the target directory either dir = Path.join([cfg.root, pfx, "aa", "aa"])
case File.ls(dir) do {:ok, entries} -> refute Enum.any?(entries, &String.contains?(&1, ".tmp."))
{:error, :enoent} -> :ok end end
test "handles large streaming write without loading into memory", %{cfg: cfg, prefix: pfx} do # 8 MB in 64 KB chunks chunk = :crypto.strong_rand_bytes(64 * 1024) chunks = List.duplicate(chunk, 128) data = IO.iodata_to_binary(chunks) oid = :crypto.hash(:sha256, data) |> Base.encode16(case: :lower)
assert {:ok, size} = Filesystem.put(cfg, pfx, oid, chunks) assert size == 8 * 1024 * 1024
{:ok, %{size: ^size, stream: stream}} = Filesystem.get(cfg, pfx, oid) read = stream |> Enum.to_list() |> IO.iodata_to_binary() assert :crypto.hash(:sha256, read) |> Base.encode16(case: :lower) == oid end end end
|
|
|
test/ex_git_objectstore/lfs/store/memory_test.exs
|
+28
−0
|
@@ -1,0 +1,28 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.MemoryTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-002"]
use ExGitObjectstore.Test.LfsStoreConformance
alias ExGitObjectstore.Lfs.Store.Memory
setup do {:ok, pid} = Memory.start_link() %{mod: Memory, cfg: Memory.config(pid), prefix: "repos/t/lfs"} end end
|
|
|
test/ex_git_objectstore/lfs/store/s3_test.exs
|
+77
−0
|
@@ -1,0 +1,77 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.Store.S3Test do use ExUnit.Case, async: false
@moduletag :s3 @moduletag requirements: ["REQ-LFS-002"]
use ExGitObjectstore.Test.LfsStoreConformance
alias ExGitObjectstore.Lfs.Store.S3
@minio_config %{ bucket: "test-bucket", ex_aws_config: [ access_key_id: "minioadmin", secret_access_key: "minioadmin", region: "us-east-1", host: "localhost", port: 9000, scheme: "http://", s3_scheme: "http://", s3_host: "localhost", s3_port: 9000 ], # 5 MiB part size — MinIO's minimum for multipart upload. Tests use # blobs small enough to take the single-part path except where noted. multipart_part_size: 5 * 1024 * 1024 }
@minio_available (case :gen_tcp.connect(~c"localhost", 9000, [], 1_000) do {:ok, socket} -> :gen_tcp.close(socket) true
{:error, _} -> false end)
if !@minio_available do @moduletag :skip end
setup do prefix = "lfs-test/#{:erlang.unique_integer([:positive])}/lfs" %{mod: S3, cfg: @minio_config, prefix: prefix} end
describe "presigned URLs" do test "presigned_upload returns an href" do oid = String.duplicate("a", 64)
assert {:ok, %{href: url, expires_in: _}} = S3.presigned_upload(@minio_config, "pfx", oid, 100, [])
assert String.contains?(url, "Amz-Signature") or String.contains?(url, "X-Amz-Signature") end
test "presigned_download returns an href" do oid = String.duplicate("b", 64) assert {:ok, %{href: url}} = S3.presigned_download(@minio_config, "pfx", oid, []) assert is_binary(url) and byte_size(url) > 0 end end end
|
|
|
test/ex_git_objectstore/lfs/transfer_test.exs
|
+134
−0
|
@@ -1,0 +1,134 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Lfs.TransferTest do use ExUnit.Case, async: true
@moduletag requirements: ["REQ-LFS-004", "REQ-LFS-006"]
alias ExGitObjectstore.Lfs.Store.Memory, as: LfsMemory alias ExGitObjectstore.Lfs.Transfer alias ExGitObjectstore.Repo alias ExGitObjectstore.Storage.Memory
setup do {:ok, git_pid} = Memory.start_link() {:ok, lfs_pid} = LfsMemory.start_link()
repo = Repo.new("r", storage: {Memory, Memory.config(git_pid)}, lfs_storage: {LfsMemory, LfsMemory.config(lfs_pid)} )
%{repo: repo} end
defp sha256_of(bin), do: :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower)
describe "upload/3 and download/2 round-trip" do test "uploads, then downloads the same bytes", %{repo: repo} do data = "hello transfer" oid = sha256_of(data)
assert {:ok, 14} = Transfer.upload(repo, oid, [data]) assert {:ok, %{size: 14, stream: stream}} = Transfer.download(repo, oid) assert IO.iodata_to_binary(Enum.to_list(stream)) == data end
test "rejects upload when sha256 does not match", %{repo: repo} do claimed = String.duplicate("a", 64) assert {:error, :oid_mismatch} = Transfer.upload(repo, claimed, ["unrelated bytes"]) end
test "download returns :not_found for missing oid", %{repo: repo} do assert {:error, :not_found} = Transfer.download(repo, String.duplicate("b", 64)) end
test "rejects bad oid format", %{repo: repo} do assert {:error, :bad_oid} = Transfer.download(repo, "bad") assert {:error, :bad_oid} = Transfer.upload(repo, "bad", ["x"]) end end
describe "verify/3" do test "returns :ok when size matches", %{repo: repo} do data = "verify me" oid = sha256_of(data) {:ok, _} = Transfer.upload(repo, oid, [data]) assert :ok = Transfer.verify(repo, oid, 9) end
test "returns :size_mismatch when wrong", %{repo: repo} do data = "verify me" oid = sha256_of(data) {:ok, _} = Transfer.upload(repo, oid, [data]) assert {:error, :size_mismatch} = Transfer.verify(repo, oid, 99) end
test "returns :not_found when absent", %{repo: repo} do assert {:error, :not_found} = Transfer.verify(repo, String.duplicate("a", 64), 1) end end
describe "when lfs not configured" do setup do {:ok, pid} = Memory.start_link() repo = Repo.new("r", storage: {Memory, Memory.config(pid)}) %{repo: repo} end
test "download returns :lfs_not_configured", %{repo: repo} do assert {:error, :lfs_not_configured} = Transfer.download(repo, String.duplicate("a", 64)) end
test "upload returns :lfs_not_configured", %{repo: repo} do assert {:error, :lfs_not_configured} = Transfer.upload(repo, String.duplicate("a", 64), ["x"]) end
test "verify returns :lfs_not_configured", %{repo: repo} do assert {:error, :lfs_not_configured} = Transfer.verify(repo, String.duplicate("a", 64), 1) end end
describe "telemetry" do test "emits span events", %{repo: repo} do parent = self() ref = make_ref()
:telemetry.attach_many( ref, [ [:ex_git_objectstore, :lfs, :transfer, :start], [:ex_git_objectstore, :lfs, :transfer, :stop] ], fn event, _m, meta, _ -> send(parent, {:tele, ref, event, meta}) end, nil )
on_exit(fn -> :telemetry.detach(ref) end)
data = "event test" oid = sha256_of(data) {:ok, _} = Transfer.upload(repo, oid, [data])
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :transfer, :start], %{operation: :upload}}
assert_receive {:tele, ^ref, [:ex_git_objectstore, :lfs, :transfer, :stop], %{operation: :upload, outcome: :ok}} end end end
|
|
|
test/support/lfs_http_adapter.ex
|
+316
−0
|
@@ -1,0 +1,316 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Test.LfsHttpAdapter do @moduledoc """ Plug router that wires the LFS library modules to HTTP endpoints so real `git lfs` clients can drive the library end-to-end.
Routes (under `/:repo_id/info/lfs`):
* `POST /objects/batch` * `GET /objects/:oid` * `PUT /objects/:oid` * `POST /verify` * `POST /locks` * `POST /locks/verify` * `POST /locks/:id/unlock` * `GET /locks`
Tests wire this via Bandit in `setup` and tear it down after. The server relies on a `:repo_factory` plug_opt — a function that takes `repo_id` and returns an `ExGitObjectstore.Repo.t()` — so each test can inject its own storage/lfs_storage configuration. """
use Plug.Router
alias ExGitObjectstore.Lfs.{Batch, Locks, Transfer} alias ExGitObjectstore.Protocol.{PktLine, ReceivePack, UploadPack}
plug(:stash_opts) plug(:match) plug(Plug.Parsers, parsers: [:json], pass: ["*/*"], json_decoder: Jason) plug(:dispatch)
defp stash_opts(conn, _), do: Plug.Conn.put_private(conn, :lfs_opts, conn.assigns[:_opts] || [])
def init(opts), do: opts
def call(conn, opts) do conn |> Plug.Conn.assign(:_opts, opts) |> super(opts) end
# -- Git smart-http (protocol v0) -- # # GET /:repo_id/info/refs?service=git-(upload|receive)-pack → service banner + advertisement # POST /:repo_id/git-(upload|receive)-pack → single request/response cycle
get "/:repo_id/info/refs" do repo = build_repo(conn, repo_id)
case conn.query_params["service"] do "git-upload-pack" -> {advert, _state} = UploadPack.init(repo) send_smart_http_advert(conn, "git-upload-pack", advert)
"git-receive-pack" -> {advert, _state} = ReceivePack.init(repo) send_smart_http_advert(conn, "git-receive-pack", advert)
_ -> send_resp(conn, 400, "unknown service") end end
post "/:repo_id/git-upload-pack" do repo = build_repo(conn, repo_id) {:ok, conn, body} = read_full_body(conn) {_advert, state} = UploadPack.init(repo) {response, _state} = UploadPack.feed(state, body)
conn |> put_resp_content_type("application/x-git-upload-pack-result") |> send_resp(200, response) end
post "/:repo_id/git-receive-pack" do repo = build_repo(conn, repo_id) {:ok, conn, body} = read_full_body(conn) {_advert, state} = ReceivePack.init(repo) {response, _state} = ReceivePack.feed(state, body)
conn |> put_resp_content_type("application/x-git-receive-pack-result") |> send_resp(200, response) end
# -- Git LFS --
post "/:repo_id/info/lfs/objects/batch" do repo = build_repo(conn, repo_id) body = conn.body_params base_url = build_base_url(conn, repo_id)
response = Batch.handle(repo, body, base_url: base_url)
conn |> put_resp_content_type("application/vnd.git-lfs+json") |> send_resp(response.status, Jason.encode!(response.body)) end
get "/:repo_id/info/lfs/objects/:oid" do repo = build_repo(conn, repo_id)
case Transfer.download(repo, oid) do {:ok, %{size: size, stream: stream}} -> conn = conn |> put_resp_header("content-length", Integer.to_string(size)) |> put_resp_content_type("application/octet-stream") |> send_chunked(200)
Enum.reduce_while(stream, conn, fn chunk, c -> case Plug.Conn.chunk(c, chunk) do {:ok, c} -> {:cont, c} {:error, _} -> {:halt, c} end end)
{:error, :not_found} -> send_lfs_error(conn, 404, "object not found")
{:error, :bad_oid} -> send_lfs_error(conn, 422, "bad oid")
{:error, :lfs_not_configured} -> send_lfs_error(conn, 501, "LFS not configured") end end
put "/:repo_id/info/lfs/objects/:oid" do repo = build_repo(conn, repo_id) {:ok, conn, stream} = collect_body_stream(conn)
case Transfer.upload(repo, oid, stream) do {:ok, _bytes} -> send_resp(conn, 200, "")
{:error, :oid_mismatch} -> send_lfs_error(conn, 422, "sha256 mismatch")
{:error, :bad_oid} -> send_lfs_error(conn, 422, "bad oid")
{:error, reason} -> send_lfs_error(conn, 500, "upload failed: #{inspect(reason)}") end end
post "/:repo_id/info/lfs/verify" do repo = build_repo(conn, repo_id) %{"oid" => oid, "size" => size} = conn.body_params
case Transfer.verify(repo, oid, size) do :ok -> send_resp(conn, 200, "") {:error, :not_found} -> send_lfs_error(conn, 404, "object not found") {:error, :size_mismatch} -> send_lfs_error(conn, 422, "size mismatch") {:error, reason} -> send_lfs_error(conn, 500, inspect(reason)) end end
post "/:repo_id/info/lfs/locks" do repo = build_repo(conn, repo_id) %{"path" => path} = conn.body_params owner = requester_name(conn)
case Locks.create(repo, path, owner) do {:ok, lock} -> send_json(conn, 201, %{"lock" => lock_to_json(lock)})
{:error, {:conflict, existing}} -> send_json(conn, 409, %{ "lock" => lock_to_json(existing), "message" => "already locked" })
{:error, :bad_request} -> send_lfs_error(conn, 400, "bad request") end end
post "/:repo_id/info/lfs/locks/verify" do repo = build_repo(conn, repo_id) owner = requester_name(conn)
case Locks.verify(repo, owner) do {:ok, %{ours: ours, theirs: theirs}} -> send_json(conn, 200, %{ "ours" => Enum.map(ours, &lock_to_json/1), "theirs" => Enum.map(theirs, &lock_to_json/1) })
{:error, reason} -> send_lfs_error(conn, 500, inspect(reason)) end end
post "/:repo_id/info/lfs/locks/:id/unlock" do repo = build_repo(conn, repo_id) force? = conn.body_params["force"] == true owner = requester_name(conn)
case Locks.unlock(repo, id, owner, force: force?) do {:ok, lock} -> send_json(conn, 200, %{"lock" => lock_to_json(lock)}) {:error, :not_found} -> send_lfs_error(conn, 404, "lock not found") {:error, :forbidden} -> send_lfs_error(conn, 403, "not the lock owner") {:error, reason} -> send_lfs_error(conn, 500, inspect(reason)) end end
get "/:repo_id/info/lfs/locks" do repo = build_repo(conn, repo_id)
opts = [] |> maybe_put(:path, conn.query_params["path"]) |> maybe_put(:id, conn.query_params["id"])
case Locks.list(repo, opts) do {:ok, locks} -> send_json(conn, 200, %{"locks" => Enum.map(locks, &lock_to_json/1)})
{:error, reason} -> send_lfs_error(conn, 500, inspect(reason)) end end
match _ do send_resp(conn, 404, "") end
# -- Helpers --
defp build_repo(conn, repo_id) do factory = Keyword.fetch!(conn.private.lfs_opts, :repo_factory) factory.(repo_id) end
defp build_base_url(conn, repo_id) do scheme = Atom.to_string(conn.scheme) port_str = if conn.port in [80, 443], do: "", else: ":#{conn.port}" "#{scheme}://#{conn.host}#{port_str}/#{repo_id}/info/lfs" end
defp send_lfs_error(conn, status, message) do conn |> put_resp_content_type("application/vnd.git-lfs+json") |> send_resp(status, Jason.encode!(%{"message" => message})) end
defp send_json(conn, status, body) do conn |> put_resp_content_type("application/vnd.git-lfs+json") |> send_resp(status, Jason.encode!(body)) end
defp collect_body_stream(conn, chunks \\ []) do case Plug.Conn.read_body(conn, length: 16 * 1024 * 1024, read_length: 16 * 1024 * 1024) do {:more, chunk, conn} -> collect_body_stream(conn, [chunk | chunks]) {:ok, chunk, conn} -> {:ok, conn, Enum.reverse([chunk | chunks])} end end
defp read_full_body(conn, chunks \\ []) do case Plug.Conn.read_body(conn, length: 64 * 1024 * 1024, read_length: 16 * 1024 * 1024) do {:more, chunk, conn} -> read_full_body(conn, [chunk | chunks]) {:ok, chunk, conn} -> {:ok, conn, IO.iodata_to_binary(Enum.reverse([chunk | chunks]))} end end
defp send_smart_http_advert(conn, service, advert) do body = IO.iodata_to_binary([ PktLine.encode("# service=#{service}\n"), PktLine.flush(), advert ])
conn |> put_resp_content_type("application/x-#{service}-advertisement") |> put_resp_header("cache-control", "no-cache") |> send_resp(200, body) end
defp requester_name(conn) do case Plug.Conn.get_req_header(conn, "x-lfs-user") do [name | _] -> name _ -> "anonymous" end end
defp lock_to_json(%{id: id, path: path, locked_at: locked_at, owner: %{name: name}}) do %{ "id" => id, "path" => path, "locked_at" => locked_at, "owner" => %{"name" => name} } end
defp maybe_put(opts, _k, nil), do: opts defp maybe_put(opts, k, v), do: Keyword.put(opts, k, v) end
|
|
|
test/support/lfs_store_conformance.ex
|
+158
−0
|
@@ -1,0 +1,158 @@ # Copyright 2026 Cole Christensen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.
defmodule ExGitObjectstore.Test.LfsStoreConformance do @moduledoc """ Shared conformance test suite for `ExGitObjectstore.Lfs.Store` backends.
Usage:
defmodule MyBackendTest do use ExGitObjectstore.Test.LfsStoreConformance
setup do {:ok, pid} = MyBackend.start_link() %{mod: MyBackend, cfg: MyBackend.config(pid), prefix: "repos/t/lfs"} end end
The `setup` must provide `%{mod: module, cfg: map, prefix: String.t()}`. """
defmacro __using__(_opts) do quote do import ExGitObjectstore.Test.LfsStoreConformance, only: [hex: 1, sha256_of: 1]
@spec_oid :crypto.hash(:sha256, "hello lfs") |> Base.encode16(case: :lower)
describe "put/4" do test "writes and verifies a small object", %{mod: mod, cfg: cfg, prefix: pfx} do data = "hello lfs" oid = sha256_of(data)
assert {:ok, 9} = mod.put(cfg, pfx, oid, [data]) assert mod.exists?(cfg, pfx, oid) end
test "rejects when observed sha256 doesn't match claimed oid", %{ mod: mod, cfg: cfg, prefix: pfx } do claimed = String.duplicate("a", 64) assert {:error, :oid_mismatch} = mod.put(cfg, pfx, claimed, ["unrelated"]) refute mod.exists?(cfg, pfx, claimed) end
test "rejects bad oid format", %{mod: mod, cfg: cfg, prefix: pfx} do assert {:error, :bad_oid} = mod.put(cfg, pfx, "not-hex", ["x"]) assert {:error, :bad_oid} = mod.put(cfg, pfx, String.duplicate("A", 64), ["x"]) end
test "accepts a chunked stream", %{mod: mod, cfg: cfg, prefix: pfx} do chunks = for i <- 1..16, do: String.duplicate("#{rem(i, 10)}", 1024) data = IO.iodata_to_binary(chunks) oid = sha256_of(data)
assert {:ok, size} = mod.put(cfg, pfx, oid, chunks) assert size == byte_size(data) end end
describe "get/3" do test "round-trips binary via stream", %{mod: mod, cfg: cfg, prefix: pfx} do data = :crypto.strong_rand_bytes(200 * 1024) oid = sha256_of(data)
{:ok, _} = mod.put(cfg, pfx, oid, [data]) {:ok, %{size: size, stream: stream}} = mod.get(cfg, pfx, oid)
assert size == byte_size(data)
read = stream |> Enum.to_list() |> IO.iodata_to_binary()
assert read == data end
test "returns :not_found for unknown oid", %{mod: mod, cfg: cfg, prefix: pfx} do assert {:error, :not_found} = mod.get(cfg, pfx, String.duplicate("f", 64)) end end
describe "stat/3 and exists?/3" do test "reports size", %{mod: mod, cfg: cfg, prefix: pfx} do data = "sized" oid = sha256_of(data) {:ok, _} = mod.put(cfg, pfx, oid, [data])
assert {:ok, %{size: 5}} = mod.stat(cfg, pfx, oid) assert mod.exists?(cfg, pfx, oid) end
test "reports not_found", %{mod: mod, cfg: cfg, prefix: pfx} do oid = String.duplicate("0", 64) assert {:error, :not_found} = mod.stat(cfg, pfx, oid) refute mod.exists?(cfg, pfx, oid) end end
describe "delete/3" do test "removes an object", %{mod: mod, cfg: cfg, prefix: pfx} do data = "deleteme" oid = sha256_of(data) {:ok, _} = mod.put(cfg, pfx, oid, [data]) assert :ok = mod.delete(cfg, pfx, oid) refute mod.exists?(cfg, pfx, oid) end
test "delete of missing is ok", %{mod: mod, cfg: cfg, prefix: pfx} do assert :ok = mod.delete(cfg, pfx, String.duplicate("0", 64)) end end
describe "list/2" do test "lists oids under prefix", %{mod: mod, cfg: cfg, prefix: pfx} do a = "a-blob" b = "b-blob" oa = sha256_of(a) ob = sha256_of(b) {:ok, _} = mod.put(cfg, pfx, oa, [a]) {:ok, _} = mod.put(cfg, pfx, ob, [b])
{:ok, oids} = mod.list(cfg, pfx) assert Enum.sort([oa, ob]) == Enum.sort(oids) end
test "isolates by prefix", %{mod: mod, cfg: cfg, prefix: pfx} do other = pfx <> "-other" data = "isolated" oid = sha256_of(data) {:ok, _} = mod.put(cfg, pfx, oid, [data])
{:ok, listed} = mod.list(cfg, other) assert listed == [] end end end end
@doc "Lowercase hex encoding helper." def hex(bin), do: Base.encode16(bin, case: :lower)
@doc "SHA256 of a binary, lowercase hex." def sha256_of(bin), do: :crypto.hash(:sha256, bin) |> hex() end
|