@@ -123,6 +123,10 @@
    duration_seconds: Option<f64>,
    runner: Option<serde_json::Value>,
    exit_code: Option<i32>,
    /// How many runners have taken a swing at this job, and what stopped the
    /// last one if the platform (rather than the code) is what failed (#375).
    attempts: Option<i32>,
    infra_failure_reason: Option<String>,
}
pub async fn run(args: CiArgs) -> Result<output::Response, Box<dyn std::error::Error>> {
@@ -348,6 +352,17 @@
    }
    if let Some(code) = job.exit_code {
        output::detail("Exit Code", &code.to_string());
    }
    // Surface an infrastructure failure as such. Before #375 this arrived as a
    // bare `exit_code: -1` and you had to go correlate it against runner logs
    // to learn the job never started.
    if let Some(ref reason) = job.infra_failure_reason {
        output::detail("Infra Failure", reason);
    }
    if let Some(n) = job.attempts {
        if n > 1 {
            output::detail("Attempts", &n.to_string());
        }
    }
    if let Some(ref ts) = job.started_at {
        output::detail("Started", &output::format_time(ts));

@@ -2077,6 +2077,31 @@
// === Admin commands (PAT auth) ===
/// How long this runner is still quarantined for, rendered for a table cell.
///
/// A quarantined runner (#375) reports `online` and heartbeats normally — a
/// host whose Docker daemon has died heartbeats perfectly well — but claims
/// nothing. `quarantined_until` is absolute, so a timestamp in the past means
/// the cooldown has already lapsed and the runner is claiming again; rendering
/// that as "quarantined" would send an operator chasing a runner that is fine.
fn quarantine_cell(quarantined_until: Option<&str>, now: chrono::DateTime<chrono::Utc>) -> String {
    let Some(until) = quarantined_until else {
        return String::new();
    };
    let Ok(until) = chrono::DateTime::parse_from_rfc3339(until) else {
        return String::new();
    };
    let remaining = until.with_timezone(&chrono::Utc) - now;
    if remaining.num_seconds() > 0 {
        format!("{}s", remaining.num_seconds())
    } else {
        String::new()
    }
}
async fn list(
    org: Option<&str>,
    repo: Option<&str>,
@@ -2107,6 +2132,8 @@
    output::header("Runners");
    let now = chrono::Utc::now();
    let rows: Vec<Vec<String>> = runners
        .iter()
        .map(|r| {
@@ -2134,11 +2161,23 @@
                    .and_then(|v| v.as_str())
                    .unwrap_or("")
                    .to_string(),
                quarantine_cell(r.get("quarantined_until").and_then(|v| v.as_str()), now),
            ]
        })
        .collect();
    output::print_table(
    output::print_table(&["ID", "NAME", "STATUS", "LABELS", "OS", "ARCH"], &rows);
        &[
            "ID",
            "NAME",
            "STATUS",
            "LABELS",
            "OS",
            "ARCH",
            "QUARANTINED",
        ],
        &rows,
    );
    // Echo the server payload verbatim — full (untruncated) ids and all.
    Ok(output::Response::read(resp))
@@ -2177,6 +2216,16 @@
    if let Some(last_seen) = runner.get("last_heartbeat_at").and_then(|v| v.as_str()) {
        output::detail("Last seen", &output::format_time(last_seen));
    }
    let quarantine = quarantine_cell(
        runner.get("quarantined_until").and_then(|v| v.as_str()),
        chrono::Utc::now(),
    );
    if !quarantine.is_empty() {
        output::detail(
            "Quarantined",
            &format!("{quarantine} remaining — claiming no jobs (#375)"),
        );
    }
    // Echo the (unwrapped) server runner object verbatim.
    Ok(output::Response::read(runner.clone()))
@@ -2525,6 +2574,45 @@
                assert!(service_name.is_none());
            }
            _ => panic!("expected Doctor"),
        }
    }
    // A quarantined runner (#375) is `online` and heartbeating and claims
    // nothing. Reading the status column alone, an operator sees a healthy
    // worker sitting idle and has no way to ask why.
    mod quarantine_column {
        use super::super::quarantine_cell;
        use chrono::{Duration, Utc};
        #[test]
        fn a_healthy_runner_has_an_empty_cell() {
            assert_eq!(quarantine_cell(None, Utc::now()), "");
        }
        #[test]
        fn a_quarantined_runner_shows_the_time_remaining() {
            let now = Utc::now();
            let until = (now + Duration::seconds(45)).to_rfc3339();
            assert_eq!(quarantine_cell(Some(&until), now), "45s");
        }
        #[test]
        fn a_lapsed_quarantine_is_not_shown() {
            let now = Utc::now();
            let until = (now - Duration::seconds(1)).to_rfc3339();
            assert_eq!(
                quarantine_cell(Some(&until), now),
                "",
                "quarantined_until is absolute — once it is in the past the runner \
                 is claiming again, and reporting it as quarantined is a lie"
            );
        }
        #[test]
        fn an_unparseable_timestamp_is_not_invented_into_a_quarantine() {
            assert_eq!(quarantine_cell(Some("not a timestamp"), Utc::now()), "");
        }
    }
}

@@ -1,5 +1,5 @@
use crate::runner::artifacts;
use crate::runner::executor::{self, ExecResult};
use crate::runner::executor::{self, EndReason, ExecResult};
use crate::runner::heartbeat;
use crate::runner::log_reporter::LogReporter;
use crate::runner::pid_file::{self, Existing};
@@ -227,7 +227,7 @@
                // on drop even if execute_job panics.
                let _reservation = slots.reserve();
                let result = execute_job(&config, &job, slot, shutdown.clone()).await;
                report_result(&client, &config, &job_id, result).await;
                report_result(&client, &config, &job_id, &job, result).await;
                drop(_reservation);
                // Handle --once / --ephemeral
@@ -487,12 +487,138 @@
    result
}
/// Whether this job needs a container at all.
///
/// A bare step never talks to the Docker daemon, so a dead daemon can never
/// explain its failure. This asks exactly the question `executor::execute`
/// asks — it routes on `image.is_some()` — because any other rule leaves a
/// payload that goes down the docker path but can never be blamed on docker.
/// (`image: bare` is resolved to no image server-side, REQ-CI-102, so the
/// literal string never reaches a runner.)
fn job_needs_container(job: &serde_json::Value) -> bool {
    job.get("image").and_then(|v| v.as_str()).is_some()
}
/// Ask the Docker daemon whether it is there.
///
/// Split from `classify_execution_error` so the decision is testable without a
/// daemon. A missing `docker` binary counts as unreachable: a runner asked to
/// run a container image and unable to is broken either way.
async fn docker_daemon_reachable() -> bool {
    tokio::task::spawn_blocking(|| {
        std::process::Command::new("docker")
            .args(["version", "--format", "{{.Server.Version}}"])
            .output()
            .map(|out| out.status.success())
            .unwrap_or(false)
    })
    .await
    .unwrap_or(false)
}
/// The failure to classify in this outcome, if there is one.
///
/// Classifying only the `Err` arm misses the shape the 2026-07-30 outage
/// actually took. `docker run` against a dead daemon does not error: it prints
/// its connect failure and exits 125, arriving here as an ordinary
/// `Ok(ExecResult)` — and a failed `docker pull` is downgraded to a logged
/// warning before that. The only path that yields `Err` is
/// `prepare::prepare_image`, and prepare is attached server-side just for
/// pipelines that declare a top-level `prepare:` block, so for every other
/// repo the outage would reproduce unchanged.
///
/// A timeout or a runner-initiated cancel DID run, so both stay terminal and
/// are never a candidate for requeueing.
fn failure_signal(
    result: &Result<ExecResult, Box<dyn std::error::Error + Send + Sync>>,
) -> Option<String> {
    match result {
        Err(e) => Some(e.to_string()),
        Ok(r) if r.end_reason == EndReason::Finished && r.exit_code != 0 => {
            Some(format!("step exited {}", r.exit_code))
        }
        Ok(_) => None,
    }
}
/// Decide how to report a failed job, given the failure signal from
/// `failure_signal` — an executor error, or a non-zero exit.
///
/// If the step needed a container and the daemon is unreachable, nothing the
/// job was asked to do can have run: this is the platform's fault and the
/// server should requeue it on a healthy peer (#375). Otherwise it is an
/// ordinary failure and stays terminal.
///
/// Returns the status to report and, for infrastructure failures, the
/// operator-facing reason that ends up on the job.
fn classify_execution_error(
    job: &serde_json::Value,
    err: &str,
    docker_reachable: bool,
) -> (&'static str, Option<String>) {
    if job_needs_container(job) && !docker_reachable {
        (
            "infra_failed",
            Some(format!("docker daemon unreachable: {err}")),
        )
    } else {
        ("failed", None)
    }
}
async fn report_result(
    client: &reqwest::Client,
    config: &RunnerConfig,
    job_id: &str,
    job: &serde_json::Value,
    result: Result<ExecResult, Box<dyn std::error::Error + Send + Sync>>,
) {
    if let Err(ref e) = result {
        eprintln!("Job execution error: {e}");
    }
    // Any failure — an executor error OR a non-zero exit from a step that
    // needed a container — may be the daemon rather than the code. The probe
    // is async and only worth paying for on a failure, so it happens here.
    if let Some(signal) = failure_signal(&result) {
        let reachable = if job_needs_container(job) {
            docker_daemon_reachable().await
        } else {
            true
        };
        if let ("infra_failed", Some(reason)) = classify_execution_error(job, &signal, reachable) {
            eprintln!(
                "Job {} could not be started on {}: {reason}",
                &job_id[..8.min(job_id.len())],
                config.name
            );
            match update_infra_failure(client, config, job_id, &reason).await {
                InfraReport::Accepted => return,
                // The job is no longer ours to report on — it finished, was
                // cancelled, or has already been handed to a healthy peer. The
                // server has fenced us; reporting a plain failure instead would
                // just be the same stale write in a different envelope.
                InfraReport::Fenced => {
                    eprintln!(
                        "Server says this job is no longer running here; dropping the report"
                    );
                    return;
                }
                // A server that predates #375 rejects this status. Fall through
                // to an ordinary failure report rather than leaving the job
                // `running` until the watchdog times it out half an hour later —
                // deploy order shouldn't be able to strand a job.
                InfraReport::Unsupported => {
                    eprintln!("Server did not accept infra_failed; reporting as a plain failure");
                }
            }
        }
    }
    let (status, exit_code, cancel_reason) = match result {
        Ok(r) if r.end_reason.is_runner_cancel() => {
            // Runner-initiated cancel: jobs the operator yanked. Server
@@ -512,10 +638,9 @@
            // policy would mis-classify a failed-timeout as cancelled.
            (s, Some(r.exit_code), None)
        }
        Err(e) => {
            eprintln!("Job execution error: {e}");
            ("failed", Some(1), None)
        }
        // Already logged above, where the daemon probe decided this was an
        // ordinary failure rather than an infrastructure one.
        Err(_) => ("failed", Some(1), None),
    };
    let cancel_suffix = cancel_reason
@@ -529,6 +654,70 @@
    update_job_status(client, config, job_id, status, exit_code, cancel_reason).await;
}
/// What the server made of an infrastructure-failure report.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum InfraReport {
    /// Recorded. The job is requeued (or failed for good, out of attempts).
    Accepted,
    /// Refused because the job is no longer running on this runner.
    Fenced,
    /// The server does not know this status at all — it predates #375.
    Unsupported,
}
/// How to read the server's answer.
///
/// Split out as a pure mapping so the fallback rule is testable without a
/// server. Only a 422 means "this server has never heard of infra_failed";
/// a 409 is a live server deliberately refusing a stale report, and anything
/// else (auth, 5xx, a proxy) is a delivery problem a plain failure report
/// would hit identically.
fn infra_report_outcome(status: u16) -> InfraReport {
    match status {
        200..=299 => InfraReport::Accepted,
        422 => InfraReport::Unsupported,
        _ => InfraReport::Fenced,
    }
}
/// Tell the server the job never started, and why (#375).
///
/// Deliberately not routed through `update_job_status`: there is no exit code
/// to send, because nothing ran.
async fn update_infra_failure(
    client: &reqwest::Client,
    config: &RunnerConfig,
    job_id: &str,
    reason: &str,
) -> InfraReport {
    let url = config.api_url(&format!("/runners/jobs/{job_id}/status"));
    let body = serde_json::json!({
        "status": "infra_failed",
        "infra_failure_reason": reason,
    });
    let result = client
        .patch(&url)
        .header(
            AUTHORIZATION,
            HeaderValue::from_str(&config.auth_header()).unwrap(),
        )
        .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
        .json(&body)
        .send()
        .await;
    match result {
        Ok(resp) => infra_report_outcome(resp.status().as_u16()),
        Err(e) => {
            eprintln!("failed to report infrastructure failure: {e}");
            // The report never landed, so there is nothing to fall back to —
            // a plain failure report would fail the same way.
            InfraReport::Fenced
        }
    }
}
async fn update_job_status(
    client: &reqwest::Client,
    config: &RunnerConfig,
@@ -577,4 +766,189 @@
        .send()
        .await?;
    Ok(())
}
#[cfg(test)]
mod infra_failure_tests {
    use super::*;
    use serde_json::json;
    // #375: carl's Docker daemon died during a reboot while the runner process
    // kept polling. It claimed four jobs in thirteen seconds and reported every
    // one as a plain exit-1 failure, indistinguishable from "your tests broke".
    // The server had no way to tell it should requeue them.
    #[test]
    fn a_containerized_job_needs_the_daemon() {
        assert!(job_needs_container(
            &json!({"image": "hexpm/elixir:1.20.2"})
        ));
    }
    #[test]
    fn a_bare_job_does_not() {
        // What the server actually sends for a bare step: no image at all.
        // `image: bare` is resolved to nil server-side (REQ-CI-102), which is
        // why the literal string is not special-cased here — see
        // `an_empty_image_is_still_a_container_job`.
        assert!(!job_needs_container(&json!({})));
        assert!(!job_needs_container(
            &json!({"image": serde_json::Value::Null})
        ));
    }
    #[test]
    fn a_dead_daemon_under_a_container_job_is_infrastructure() {
        let (status, reason) = classify_execution_error(
            &json!({"image": "hexpm/elixir:1.20.2"}),
            "Prepare failed: Prepare commands failed with exit code 125",
            false,
        );
        assert_eq!(
            status, "infra_failed",
            "a job that never started must not be reported as a code failure"
        );
        let reason = reason.expect("infra failures must carry an operator-facing reason");
        assert!(
            reason.contains("docker daemon unreachable"),
            "reason should name the cause, got: {reason}"
        );
        assert!(
            reason.contains("exit code 125"),
            "reason should keep the underlying error, got: {reason}"
        );
    }
    #[test]
    fn a_live_daemon_means_the_job_itself_failed() {
        let (status, reason) = classify_execution_error(
            &json!({"image": "hexpm/elixir:1.20.2"}),
            "step exited 1",
            true,
        );
        assert_eq!(status, "failed");
        assert!(reason.is_none());
    }
    #[test]
    fn a_bare_job_is_never_blamed_on_docker() {
        let (status, reason) = classify_execution_error(&json!({}), "script blew up", false);
        assert_eq!(
            status, "failed",
            "a bare step never touches the daemon, so a dead daemon cannot excuse it"
        );
        assert!(reason.is_none());
    }
    // The outage did NOT arrive on the Err arm. `docker run` against a dead
    // daemon prints its connect error and exits 125, which comes back as an
    // ordinary `Ok(ExecResult)`; a failed `docker pull` is only logged as a
    // warning. The one path that errors is `prepare::prepare_image`, and
    // prepare is attached server-side only for pipelines that declare a
    // top-level `prepare:` block — so for every other repo the 2026-07-30
    // outage reproduced unchanged.
    fn finished(exit_code: i32) -> Result<ExecResult, Box<dyn std::error::Error + Send + Sync>> {
        Ok(ExecResult {
            exit_code,
            end_reason: EndReason::Finished,
        })
    }
    #[test]
    fn a_nonzero_exit_is_a_failure_signal() {
        let signal = failure_signal(&finished(125)).expect("exit 125 is a failure");
        assert!(
            signal.contains("125"),
            "the signal should carry the exit code, got: {signal}"
        );
    }
    #[test]
    fn a_dead_daemon_behind_exit_125_is_infrastructure() {
        let signal = failure_signal(&finished(125)).expect("exit 125 is a failure");
        let (status, reason) =
            classify_execution_error(&json!({"image": "hexpm/elixir:1.20.2"}), &signal, false);
        assert_eq!(
            status, "infra_failed",
            "docker run against a dead daemon exits 125 — the shape the outage actually took"
        );
        assert!(reason
            .expect("reason")
            .contains("docker daemon unreachable"));
    }
    #[test]
    fn a_passing_job_is_not_a_failure_signal() {
        assert!(failure_signal(&finished(0)).is_none());
    }
    #[test]
    fn a_timed_out_job_is_not_a_failure_signal() {
        let timed_out = Ok(ExecResult {
            exit_code: 124,
            end_reason: EndReason::Timeout,
        });
        assert!(
            failure_signal(&timed_out).is_none(),
            "a job that outlived its own timeout did run; timeouts stay terminal"
        );
    }
    #[test]
    fn a_runner_cancel_is_not_a_failure_signal() {
        let cancelled = Ok(ExecResult {
            exit_code: 130,
            end_reason: EndReason::RunnerShutdown,
        });
        assert!(
            failure_signal(&cancelled).is_none(),
            "the operator stopped this runner; that is not the job failing"
        );
    }
    #[test]
    fn an_execution_error_is_still_a_failure_signal() {
        let err: Result<ExecResult, Box<dyn std::error::Error + Send + Sync>> =
            Err("Prepare failed: Prepare commands failed with exit code 125".into());
        assert_eq!(
            failure_signal(&err).as_deref(),
            Some("Prepare failed: Prepare commands failed with exit code 125")
        );
    }
    #[test]
    fn only_a_422_means_the_server_predates_this() {
        assert_eq!(infra_report_outcome(200), InfraReport::Accepted);
        assert_eq!(
            infra_report_outcome(422),
            InfraReport::Unsupported,
            "422 is the old server rejecting an unknown status; fall back to a plain failure"
        );
        assert_eq!(
            infra_report_outcome(409),
            InfraReport::Fenced,
            "409 is a live server refusing a stale report — retrying it as a plain \
             failure is the same stale write in a different envelope"
        );
        assert_eq!(infra_report_outcome(403), InfraReport::Fenced);
        assert_eq!(infra_report_outcome(500), InfraReport::Fenced);
    }
    #[test]
    fn an_empty_image_is_still_a_container_job() {
        // `executor::execute` routes on `image.is_some()`, so an empty string
        // goes down the docker path. Classification must ask the same question
        // the executor asked, or a job that only ever touched the daemon is
        // never blamed on it.
        assert!(job_needs_container(&json!({"image": ""})));
    }
}