ref:0309aab6f5cccf63a1ba7843d93b18e8cf933829

Wire prepare block into release runner binary (#7)

## Summary - Commit `src/runner/prepare.rs` (previously untracked) and declare `pub mod prepare` in `src/runner/mod.rs`. - In `execute_job`, read the `prepare` array from the job JSON; when non-empty, call `prepare::prepare_image(image, &prepare, &log_reporter)` and use the returned tag for `executor::execute`. - Fail loudly if prepare is requested but can't run (e.g. prepare without image, or docker commit error). Silent fallback to the bare base image is what produced the exit-127 failure on main. - Call `prepare::prune_prepared_images(30, 10)` once at runner start to keep cached images under control. ## Why The server has been sending `prepare: [...]` in every job JSON, and the pipeline parser has validated the block since Mar 14. The runner never implemented the counterpart — the prepare field was read nowhere. Production CI only worked because carl had a locally-built binary with an uncommitted `prepare.rs` wired in. `install.sh` migrating carl to the latest CalVer release silently traded that away, and the next main run failed with exit 127 because `git config` couldn't run in the bare `hexpm/elixir` image. ## Test plan - [x] cargo build --release — clean - [x] cargo test — 6 passed (4 new prepare tests) - [x] cargo clippy --all-targets -- -D warnings — clean - [x] cargo fmt -- --check — clean - [ ] Cut a new CalVer release from this branch. - [ ] Reinstall runner on carl and re-run the failing deps job for main — deps should now pass via a freshly-built prepared image. Closes #49
SHA: 0309aab6f5cccf63a1ba7843d93b18e8cf933829
Author: Anvil <noreply@anvil.fangorn.io>
Date: 2026-04-17 20:19
Parents: 12ff11a
5 files changed +396 -1
Type
Cargo.lock +72 −0
@@ -77,6 +77,7 @@
"reqwest",
"serde",
"serde_json",
"sha2",
"tabled",
"thiserror 2.0.18",
"tokio",
@@ -115,6 +116,15 @@
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -243,6 +253,25 @@
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "dialoguer"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -256,6 +285,16 @@
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "dirs"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -431,6 +470,16 @@
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1336,6 +1385,17 @@
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shell-words"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1662,6 +1722,12 @@
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1714,6 +1780,12 @@
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "want"
Cargo.toml +1 −0
@@ -27,3 +27,4 @@
libc = "0.2"
futures = "0.3"
glob = "0.3"
sha2 = "0.10"
src/runner/loop_runner.rs +51 −1
@@ -2,6 +2,7 @@
use crate::runner::executor::{self, ExecResult};
use crate::runner::heartbeat;
use crate::runner::log_reporter::LogReporter;
use crate::runner::prepare;
use crate::runner::service_manager;
use crate::runner::workspace;
use crate::runner::RunnerConfig;
@@ -20,5 +21,9 @@
heartbeat::send_once(&config).await?;
eprintln!("Connected. Runner '{}' ready.", config.name);
// Prune prepared images accumulated from past runs — keep 10 most recent,
// drop anything older than 30 days.
prepare::prune_prepared_images(30, 10);
// Start heartbeat
let heartbeat_handle = heartbeat::start(&config);
@@ -172,6 +177,15 @@
.and_then(|v| v.as_str())
.ok_or("job missing 'command'")?;
let image = job.get("image").and_then(|v| v.as_str());
let prepare_cmds: Vec<String> = job
.get("prepare")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default();
let repo_url = job.get("repo_clone_url").and_then(|v| v.as_str());
let commit_sha = job.get("commit_sha").and_then(|v| v.as_str());
let timeout_seconds = job.get("timeout_seconds").and_then(|v| v.as_u64());
@@ -246,10 +260,46 @@
}
});
// Build (or reuse) a prepared image if the pipeline specified a prepare block.
// The prepared image bakes apt-get/rustup/etc. so each step doesn't re-install
// its toolchain. Fail loudly if prepare is requested but can't complete —
// silently falling back to the bare base image is what caused issue #49.
let prepared_tag: Option<String> = if !prepare_cmds.is_empty() {
match image {
Some(base) => match prepare::prepare_image(base, &prepare_cmds, &log_reporter).await {
Ok(tag) => Some(tag),
Err(e) => {
log_reporter.append(&format!("Prepare failed: {e}")).await;
log_reporter.flush().await;
if let Some(ctx) = &svc_context {
service_manager::cleanup_services(&ctx.containers, &ctx.network);
}
flush_handle.abort();
return Err(e);
}
},
None => {
let msg =
"Prepare block was provided but step has no 'image' — prepare requires a container image".to_string();
log_reporter.append(&msg).await;
log_reporter.flush().await;
if let Some(ctx) = &svc_context {
service_manager::cleanup_services(&ctx.containers, &ctx.network);
}
flush_handle.abort();
return Err(msg.into());
}
}
} else {
None
};
let effective_image = prepared_tag.as_deref().or(image);
// Execute
let result = executor::execute(
command,
effective_image,
image,
ws,
&env,
network,
src/runner/mod.rs +1 −0
@@ -4,5 +4,6 @@
pub mod heartbeat;
pub mod log_reporter;
pub mod loop_runner;
pub mod prepare;
pub mod service_manager;
pub mod workspace;
src/runner/prepare.rs +271 −0
@@ -1,0 +1,271 @@
use crate::runner::log_reporter::LogReporter;
use sha2::{Digest, Sha256};
use std::process::Command;
use tokio::io::{AsyncBufReadExt, BufReader};
/// Compute a deterministic image tag from the base image and prepare commands.
pub fn compute_prepared_tag(image: &str, prepare: &[String]) -> String {
let mut hasher = Sha256::new();
hasher.update(image.as_bytes());
for cmd in prepare {
hasher.update(b"\n");
hasher.update(cmd.as_bytes());
}
let hash = hasher.finalize();
let hex = format!("{hash:x}");
format!("anvil-prepared:{}", &hex[..12])
}
/// Check if a Docker image exists locally.
fn image_exists_locally(tag: &str) -> bool {
Command::new("docker")
.args(["image", "inspect", tag])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
/// Prepare a Docker image by running prepare commands on the base image.
/// Returns the tag of the prepared image (either cached or freshly built).
pub async fn prepare_image(
image: &str,
prepare: &[String],
log_reporter: &LogReporter,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let tag = compute_prepared_tag(image, prepare);
if image_exists_locally(&tag) {
let msg = format!("Using cached prepared image: {tag}");
eprintln!("{msg}");
log_reporter.append(&msg).await;
return Ok(tag);
}
log_reporter
.append(&format!("Building prepared image from {image}..."))
.await;
// Pull the base image first
eprintln!("Pulling base image: {image}");
log_reporter
.append(&format!("Pulling base image: {image}"))
.await;
let pull_output = Command::new("docker").args(["pull", image]).output();
match pull_output {
Ok(output) if output.status.success() => {
eprintln!("Base image ready: {image}");
}
Ok(output) => {
let stderr = String::from_utf8_lossy(&output.stderr);
let msg = format!("Warning: docker pull failed: {stderr}");
eprintln!("{msg}");
log_reporter.append(&msg).await;
}
Err(e) => {
let msg = format!("Warning: docker pull error: {e}");
eprintln!("{msg}");
log_reporter.append(&msg).await;
}
}
// Join all prepare commands with && so they run in sequence and fail fast
let combined = prepare.join(" && ");
let container_name = format!("anvil-prepare-{}", &tag["anvil-prepared:".len()..]);
// Remove any leftover container from a previous failed attempt
let _ = Command::new("docker")
.args(["rm", "-f", &container_name])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
// Run prepare commands in the base image, streaming output
log_reporter.append("Running prepare commands...").await;
let mut child = tokio::process::Command::new("docker")
.args([
"run",
"--name",
&container_name,
image,
"/bin/sh",
"-c",
&combined,
])
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()?;
// Stream stdout and stderr
let reporter_out = log_reporter.clone();
let stdout = child.stdout.take();
let stdout_task = tokio::spawn(async move {
if let Some(stdout) = stdout {
let mut lines = BufReader::new(stdout).lines();
while let Ok(Some(line)) = lines.next_line().await {
reporter_out.append(&line).await;
}
}
});
let reporter_err = log_reporter.clone();
let stderr = child.stderr.take();
let stderr_task = tokio::spawn(async move {
if let Some(stderr) = stderr {
let mut lines = BufReader::new(stderr).lines();
while let Ok(Some(line)) = lines.next_line().await {
reporter_err.append(&line).await;
}
}
});
let _ = stdout_task.await;
let _ = stderr_task.await;
let status = child.wait().await?;
log_reporter.flush().await;
if !status.success() {
// Clean up failed container
let _ = Command::new("docker")
.args(["rm", "-f", &container_name])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
return Err(format!(
"Prepare commands failed with exit code {}",
status.code().unwrap_or(1)
)
.into());
}
// Commit the container as the prepared image
log_reporter
.append(&format!("Committing prepared image as {tag}..."))
.await;
let commit_output = Command::new("docker")
.args(["commit", &container_name, &tag])
.output()?;
// Remove the build container
let _ = Command::new("docker")
.args(["rm", "-f", &container_name])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
if !commit_output.status.success() {
let stderr = String::from_utf8_lossy(&commit_output.stderr);
return Err(format!("docker commit failed: {stderr}").into());
}
let msg = format!("Prepared image ready: {tag}");
eprintln!("{msg}");
log_reporter.append(&msg).await;
Ok(tag)
}
/// Prune old prepared images beyond max_count or older than max_age_days.
pub fn prune_prepared_images(max_age_days: u64, max_count: usize) {
let output = Command::new("docker")
.args([
"images",
"anvil-prepared",
"--format",
"{{.ID}} {{.CreatedAt}} {{.Repository}}:{{.Tag}}",
])
.output();
let output = match output {
Ok(o) if o.status.success() => o,
_ => return,
};
let stdout = String::from_utf8_lossy(&output.stdout);
let mut images: Vec<(&str, &str)> = stdout
.lines()
.filter_map(|line| {
let parts: Vec<&str> = line.splitn(3, ' ').collect();
if parts.len() >= 3 {
Some((parts[2], parts[1]))
} else {
None
}
})
.collect();
// Sort by creation date descending (newest first) — lexicographic on ISO date works
images.sort_by(|a, b| b.1.cmp(a.1));
// Remove images beyond max_count
for (tag, _) in images.iter().skip(max_count) {
eprintln!("Pruning old prepared image: {tag}");
let _ = Command::new("docker")
.args(["rmi", tag])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
}
// Also remove images older than max_age_days
if max_age_days > 0 {
let cutoff = chrono::Utc::now() - chrono::Duration::days(max_age_days as i64);
let cutoff_str = cutoff.format("%Y-%m-%d").to_string();
for (tag, created_at) in &images {
if *created_at < cutoff_str.as_str() {
eprintln!("Pruning expired prepared image: {tag}");
let _ = Command::new("docker")
.args(["rmi", tag])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_compute_prepared_tag_deterministic() {
let tag1 = compute_prepared_tag(
"elixir:1.17",
&["apt-get update".into(), "mix local.hex --force".into()],
);
let tag2 = compute_prepared_tag(
"elixir:1.17",
&["apt-get update".into(), "mix local.hex --force".into()],
);
assert_eq!(tag1, tag2);
assert!(tag1.starts_with("anvil-prepared:"));
assert_eq!(tag1.len(), "anvil-prepared:".len() + 12);
}
#[test]
fn test_compute_prepared_tag_differs_on_image() {
let tag1 = compute_prepared_tag("elixir:1.17", &["apt-get update".into()]);
let tag2 = compute_prepared_tag("elixir:1.18", &["apt-get update".into()]);
assert_ne!(tag1, tag2);
}
#[test]
fn test_compute_prepared_tag_differs_on_commands() {
let tag1 = compute_prepared_tag("elixir:1.17", &["apt-get update".into()]);
let tag2 = compute_prepared_tag("elixir:1.17", &["apt-get install -y git".into()]);
assert_ne!(tag1, tag2);
}
#[test]
fn test_compute_prepared_tag_order_matters() {
let tag1 = compute_prepared_tag("elixir:1.17", &["cmd1".into(), "cmd2".into()]);
let tag2 = compute_prepared_tag("elixir:1.17", &["cmd2".into(), "cmd1".into()]);
assert_ne!(tag1, tag2);
}
}