@@ -61,12 +61,34 @@
        /// before forcibly terminating them
        #[arg(long, default_value_t = runner::loop_runner::DEFAULT_SHUTDOWN_TIMEOUT_SECS)]
        shutdown_timeout: u64,
        /// Override the config's `parallel` value. Used by service
        /// units to bake the slot count into ExecStart so it's
        /// self-describing without re-reading config.json.
        #[arg(long)]
        parallel: Option<u32>,
        /// Fork into the background and redirect stdout/stderr to a log
        /// file (default ~/.anvil-runner/runner.log). Parent prints the
        /// daemon's PID and returns. Use `anvil runner stop` to stop it.
        #[arg(long)]
        detach: bool,
        /// Log file path for --detach (default ~/.anvil-runner/runner.log)
        #[arg(long)]
        log_file: Option<String>,
        /// Hidden marker that the runner is itself the detached daemon —
        /// stops the daemon trying to re-detach in an infinite loop.
        #[arg(long, hide = true)]
        detach_child: bool,
    },
    /// Stop a running runner gracefully
    Stop {
        /// PID file path. Defaults to the instance-aware path:
        /// `~/.anvil-runner/runner.pid` for the default instance,
        /// `~/.anvil-runner/runner-<name>.pid` for named instances.
        /// PID file path (default ~/.anvil-runner/runner.pid)
        #[arg(long)]
        pid_file: Option<String>,
        /// Logical service instance to target (default "default")
        #[arg(long)]
        service_name: Option<String>,
        /// Wait this many seconds for clean exit before SIGKILL (with --force)
        /// or returning an error
        #[arg(long, default_value_t = runner::loop_runner::DEFAULT_SHUTDOWN_TIMEOUT_SECS + 10)]
@@ -80,19 +102,59 @@
        /// Config file path
        #[arg(long)]
        config: Option<String>,
        /// PID file path (default ~/.anvil-runner/runner.pid)
        /// PID file path (default instance-aware, see `runner stop --pid-file`)
        #[arg(long)]
        pid_file: Option<String>,
        /// Logical service instance to target (default "default")
        #[arg(long)]
        service_name: Option<String>,
        /// Stop timeout (see `runner stop --timeout`)
        #[arg(long, default_value_t = runner::loop_runner::DEFAULT_SHUTDOWN_TIMEOUT_SECS + 10)]
        timeout: u64,
    },
    /// Show runner status: config, service install, process running,
    /// Show runner configuration
    /// last heartbeat. Reads local state only — no server calls.
    Status {
        /// Config file path
        #[arg(long)]
        config: Option<String>,
        /// PID file path (default ~/.anvil-runner/runner.pid)
        #[arg(long)]
        pid_file: Option<String>,
        /// Logical service instance to inspect (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Tail the runner's logs (auto-detects service mode, falls back to
    /// the --detach log file).
    Logs {
        /// Follow new output (`tail -f` / `journalctl -f`).
        #[arg(long, short = 'f')]
        follow: bool,
        /// Show only the last N lines.
        #[arg(long, short = 'n', default_value_t = 100)]
        lines: u32,
        /// Logical service instance (default "default")
        #[arg(long)]
        service_name: Option<String>,
        /// Override log file path (used when no service is installed
        /// and the runner is running detached).
        #[arg(long)]
        log_file: Option<String>,
    },
    /// Run a series of diagnostic checks against the local install,
    /// the server, and the running process.
    Doctor {
        /// Config file path
        #[arg(long)]
        config: Option<String>,
        /// PID file path (default ~/.anvil-runner/runner.pid)
        #[arg(long)]
        pid_file: Option<String>,
        /// Logical service instance to check (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Deregister from server and remove config
    Unconfigure {
        /// Config file path
@@ -185,21 +247,48 @@
        /// value in config.json.
        #[arg(long)]
        parallel: Option<u32>,
        /// Logical instance name. Use to install multiple distinct
        /// runners (e.g., `gpu1`, `cpu-only`) on one host. Defaults to
        /// "default". Must match ^[a-zA-Z0-9][a-zA-Z0-9_-]*$.
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Uninstall the system service
    Uninstall,
    Uninstall {
        /// Logical instance name (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Start the service
    #[command(name = "svc-start")]
    Start {
    Start,
        /// Logical instance name (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Stop the service
    #[command(name = "svc-stop")]
    Stop,
    Stop {
        /// Logical instance name (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Restart the service (for picking up an upgraded binary)
    #[command(name = "svc-restart")]
    Restart,
    Restart {
        /// Logical instance name (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// Show service status
    #[command(name = "svc-status")]
    SvcStatus,
    SvcStatus {
        /// Logical instance name (default "default")
        #[arg(long)]
        service_name: Option<String>,
    },
    /// List installed runner service instances on this host
    List,
}
pub async fn run(args: RunnerArgs) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -234,27 +323,67 @@
            config,
            pid_file,
            shutdown_timeout,
            parallel,
            detach,
            log_file,
            detach_child,
        } => {
            start(StartOpts {
            start(
                once,
                ephemeral,
                cleanup: &cleanup,
                &cleanup,
                config.as_deref(),
                config_path: config.as_deref(),
                pid_file: pid_file.as_deref(),
                pid_file.as_deref(),
                shutdown_timeout,
                parallel,
                detach,
            )
                log_file: log_file.as_deref(),
                detach_child,
            })
            .await
        }
        RunnerCommand::Stop {
            pid_file,
            service_name,
            timeout,
            force,
        } => {
            let instance = resolve_instance(service_name)?;
        } => stop(pid_file.as_deref(), timeout, force).await,
            stop(pid_file.as_deref(), &instance, timeout, force).await
        }
        RunnerCommand::Restart {
            config,
            pid_file,
            service_name,
            timeout,
        } => {
            let instance = resolve_instance(service_name)?;
            restart(config.as_deref(), pid_file.as_deref(), &instance, timeout).await
        }
        RunnerCommand::Status {
            config,
            pid_file,
            service_name,
        } => {
            let instance = resolve_instance(service_name)?;
            status(config.as_deref(), pid_file.as_deref(), &instance).await
        }
        RunnerCommand::Logs {
            follow,
            lines,
            service_name,
            log_file,
        } => {
            let instance = resolve_instance(service_name)?;
            logs(follow, lines, &instance, log_file.as_deref()).await
        }
        RunnerCommand::Doctor {
            config,
            pid_file,
        } => restart(config.as_deref(), pid_file.as_deref(), timeout).await,
        RunnerCommand::Status { config } => status(config.as_deref()).await,
            service_name,
        } => {
            let instance = resolve_instance(service_name)?;
            doctor(config.as_deref(), pid_file.as_deref(), &instance).await
        }
        RunnerCommand::Unconfigure { config } => unconfigure(config.as_deref()).await,
        RunnerCommand::Service(svc) => service(svc).await,
@@ -390,12 +519,13 @@
async fn stop(
    pid_file: Option<&str>,
    instance: &str,
    timeout_secs: u64,
    force: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let pid_path = pid_file
        .map(std::path::PathBuf::from)
        .unwrap_or_else(runner::pid_file::default_path);
        .unwrap_or_else(|| runner::pid_file::instance_path(instance));
    match runner::pid_file::inspect(&pid_path) {
        runner::pid_file::Existing::None => {
@@ -468,12 +598,13 @@
async fn restart(
    config_path: Option<&str>,
    pid_file: Option<&str>,
    instance: &str,
    timeout_secs: u64,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    // Stop is no-op if nothing's running. Honor --force semantics by
    // defaulting to true on restart — if a binary upgrade caller can't
    // get the old version to stop cleanly, escalating is the right call.
    stop(pid_file, instance, timeout_secs, true).await?;
    stop(pid_file, timeout_secs, true).await?;
    // Re-exec ourselves with the same config so the NEW binary on disk
    // takes over. We use exec rather than spawn because restart's whole
@@ -487,6 +618,11 @@
    }
    if let Some(p) = pid_file {
        cmd.arg("--pid-file").arg(p);
    } else if instance != runner::service_mode::DEFAULT_INSTANCE {
        // Carry the instance forward so the new runner uses the same
        // PID file the stop just freed up.
        let path = runner::pid_file::instance_path(instance);
        cmd.arg("--pid-file").arg(path);
    }
    #[cfg(unix)]
@@ -502,18 +638,108 @@
    }
}
async fn start(
struct StartOpts<'a> {
    once: bool,
    ephemeral: bool,
    cleanup: &str,
    config_path: Option<&str>,
    pid_file: Option<&str>,
    cleanup: &'a str,
    config_path: Option<&'a str>,
    pid_file: Option<&'a str>,
    shutdown_timeout: u64,
    parallel: Option<u32>,
    detach: bool,
    log_file: Option<&'a str>,
    detach_child: bool,
}
async fn start(opts: StartOpts<'_>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let StartOpts {
        once,
        ephemeral,
        cleanup,
        config_path,
        pid_file,
        shutdown_timeout,
        parallel,
        detach,
        log_file,
        detach_child,
    } = opts;
    // If --detach is set AND we're not already the daemon: re-exec
    // ourselves with stdio redirected to the log file, then exit. The
    // daemon child runs the rest of this function with --detach-child
    // set, which short-circuits the re-detach.
    if detach && !detach_child {
        let log_path = log_file
            .map(std::path::PathBuf::from)
            .unwrap_or_else(runner::detach::default_log_path);
        let mut forwarded: Vec<String> = Vec::new();
        if once {
            forwarded.push("--once".into());
        }
        if ephemeral {
            forwarded.push("--ephemeral".into());
        }
        if cleanup != "never" {
            forwarded.push("--cleanup".into());
            forwarded.push(cleanup.into());
        }
        if let Some(c) = config_path {
            forwarded.push("--config".into());
            forwarded.push(c.into());
        }
        if let Some(p) = pid_file {
            forwarded.push("--pid-file".into());
            forwarded.push(p.into());
        }
        if let Some(p) = parallel {
            forwarded.push("--parallel".into());
            forwarded.push(p.to_string());
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        }
        forwarded.push("--shutdown-timeout".into());
        forwarded.push(shutdown_timeout.to_string());
        let pid_path = pid_file
            .map(std::path::PathBuf::from)
            .unwrap_or_else(runner::pid_file::default_path);
        let pid = runner::detach::spawn_daemon(&log_path, &forwarded)?;
        // Wait for the daemon child to come up — either acquire its PID
        // file (success) or die (failure). Bounded so a hung child
        // doesn't block the parent forever; the operator can `runner
        // doctor` if it times out.
        match runner::detach::wait_for_daemon_ready(pid, &pid_path, &log_path)? {
            runner::detach::DaemonReady::Live(child_pid) => {
                output::success(&format!(
                    "Runner detached (PID {child_pid}). Logs: {}",
                    log_path.display()
                ));
            }
            runner::detach::DaemonReady::DiedDuringStartup(tail) => {
                return Err(format!(
                    "Daemon (PID {pid}) died during startup. Last log lines:\n{tail}"
                )
                .into());
            }
            runner::detach::DaemonReady::Timeout => {
                output::warn(&format!(
                    "Daemon (PID {pid}) didn't write a PID file within startup window. \
                     Check {}",
                    log_path.display()
                ));
            }
        }
        return Ok(());
    }
    let mut config = RunnerConfig::load(config_path)?;
    config.once = once;
    config.ephemeral = ephemeral;
    config.cleanup = cleanup.to_string();
    if let Some(p) = parallel {
        config.parallel = p;
    }
    let pid_path = pid_file
        .map(std::path::PathBuf::from)
@@ -522,29 +748,335 @@
    runner::loop_runner::start(config, pid_path, shutdown_timeout).await
}
async fn status(
    config_path: Option<&str>,
    pid_file: Option<&str>,
    instance: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let config = RunnerConfig::load(config_path).ok();
    let install = runner::service_mode::load(instance);
async fn status(config_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let pid_path = pid_file
        .map(std::path::PathBuf::from)
        .unwrap_or_else(runner::pid_file::default_path);
    let pid_state = runner::pid_file::inspect(&pid_path);
    let config = RunnerConfig::load(config_path)?;
    output::header("Runner Configuration");
    output::detail("Server", &config.server_url);
    output::detail("Runner ID", &config.runner_id);
    output::detail("Name", &config.name);
    output::detail("Labels", &config.labels.join(", "));
    output::detail("Work dir", &config.work_dir);
    output::detail("Parallel", &config.parallel.to_string());
    output::detail("Poll interval", &format!("{}ms", config.poll_interval_ms));
    let json_mode = output::is_json();
    if json_mode {
        let body = serde_json::json!({
            "configured": config.is_some(),
            "config_path": RunnerConfig::path(config_path).display().to_string(),
            "config": config.as_ref().map(|c| serde_json::json!({
                "server": c.server_url,
                "runner_id": c.runner_id,
                "name": c.name,
                "labels": c.labels,
                "work_dir": c.work_dir,
                "parallel": c.parallel,
            })),
            "service": install.as_ref().map(|r| serde_json::json!({
                "instance": r.instance,
                "scope": r.scope.as_str(),
                "unit_path": r.unit_path.display().to_string(),
                "unit_or_label": r.unit_or_label,
                "parallel": r.parallel,
                "user_account": r.user_account,
            })),
            "running": match pid_state {
                runner::pid_file::Existing::Live(pid) => serde_json::json!({
                    "alive": true,
                    "pid": pid,
                    "pid_file": pid_path.display().to_string(),
                }),
                runner::pid_file::Existing::Stale(pid) => serde_json::json!({
                    "alive": false,
                    "stale_pid": pid,
                    "pid_file": pid_path.display().to_string(),
                }),
                runner::pid_file::Existing::None => serde_json::json!({
                    "alive": false,
                    "pid_file": pid_path.display().to_string(),
                }),
            },
        });
        output::print_json(&body);
        return Ok(());
    }
    output::header("Runner Status");
    output::detail("Configured", if config.is_some() { "yes" } else { "no" });
    if let Some(ref c) = config {
        output::detail("Server", &c.server_url);
        output::detail("Runner ID", &c.runner_id);
        output::detail("Name", &c.name);
        output::detail("Labels", &c.labels.join(", "));
        output::detail("Parallel", &c.parallel.to_string());
    }
    output::detail(
        "Service installed",
        &match install {
            Some(ref r) => format!(
                "yes ({} scope, instance '{}', unit {})",
                r.scope.as_str(),
                r.instance,
                r.unit_or_label
            ),
            None => "no".to_string(),
        },
        "Heartbeat interval",
        &format!("{}ms", config.heartbeat_interval_ms),
    );
    output::detail(
        "Process running",
        &match pid_state {
        "Config",
        &RunnerConfig::path(config_path).display().to_string(),
            runner::pid_file::Existing::Live(pid) => format!("yes (PID {pid})"),
            runner::pid_file::Existing::Stale(pid) => {
                format!("no (stale PID {pid} in {})", pid_path.display())
            }
            runner::pid_file::Existing::None => "no (no PID file)".to_string(),
        },
    );
    Ok(())
}
async fn logs(
    follow: bool,
    lines: u32,
    instance: &str,
    log_file_override: Option<&str>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    // Explicit log-file override wins (handy for --detach with custom path).
    if let Some(path) = log_file_override {
        return tail_file(std::path::Path::new(path), follow, lines);
    }
    // If a service is installed, route by mode.
    if let Some(record) = runner::service_mode::load(instance) {
        if cfg!(target_os = "macos") {
            // Look up the StandardOutPath we wrote into the plist by
            // instance. Pattern matches install_launchd().
            let log = match record.scope {
                runner::service_mode::Scope::User => dirs::home_dir()
                    .unwrap()
                    .join("Library/Logs/anvil-runner")
                    .join(format!("{instance}.log")),
                runner::service_mode::Scope::System => {
                    std::path::PathBuf::from("/var/log/anvil-runner")
                        .join(format!("{instance}.log"))
                }
            };
            return tail_file(&log, follow, lines);
        } else {
            // systemd: journalctl
            let unit = record
                .unit_or_label
                .trim_end_matches(".service")
                .to_string();
            let mut args: Vec<String> = Vec::new();
            if matches!(record.scope, runner::service_mode::Scope::User) {
                args.push("--user".into());
            }
            args.push("-u".into());
            args.push(unit);
            args.push("-n".into());
            args.push(lines.to_string());
            if follow {
                args.push("-f".into());
            }
            let status = std::process::Command::new("journalctl")
                .args(&args)
                .status();
            match status {
                Ok(s) if s.success() => return Ok(()),
                Ok(s) => return Err(format!("journalctl exited {s}").into()),
                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
                    return Err("`journalctl` not found on PATH; install systemd or \
                         use `--log-file <path>` to read a detach log directly"
                        .into());
                }
                Err(e) => return Err(format!("failed to exec journalctl: {e}").into()),
            }
        }
    }
    // No service install — try the --detach default log file.
    let detach_log = runner::detach::default_log_path();
    if detach_log.exists() {
        return tail_file(&detach_log, follow, lines);
    }
    Err(format!(
        "no service install for '{instance}' and no detach log file at {}; \
         logs from a foreground `runner start` go to your terminal",
        detach_log.display()
    )
    .into())
}
fn tail_file(
    path: &std::path::Path,
    follow: bool,
    lines: u32,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    if !path.exists() {
        return Err(format!("log file not found: {}", path.display()).into());
    }
    let mut args: Vec<String> = vec!["-n".into(), lines.to_string()];
    if follow {
        args.push("-f".into());
    }
    args.push(path.to_string_lossy().into_owned());
    let status = std::process::Command::new("tail").args(&args).status();
    match status {
        Ok(s) if s.success() => Ok(()),
        Ok(s) => Err(format!("tail exited {s}").into()),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
            Err("`tail` not found on PATH; cannot stream the log file. \
             On a minimal container, install coreutils or use \
             `cat <log-path>` directly."
                .into())
        }
        Err(e) => Err(format!("failed to exec tail: {e}").into()),
    }
}
async fn doctor(
    config_path: Option<&str>,
    pid_file: Option<&str>,
    instance: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let mut checks: Vec<(String, Result<String, String>)> = Vec::new();
    // 1. Config loadable
    let config_result = RunnerConfig::load(config_path);
    let config_ok = config_result.is_ok();
    checks.push((
        "Config readable".into(),
        match &config_result {
            Ok(c) => Ok(format!(
                "{} (server={})",
                RunnerConfig::path(config_path).display(),
                c.server_url
            )),
            Err(e) => Err(format!("{e}")),
        },
    ));
    // 2. Server reachable + token works. Use the read-only `GET
    //    /runners/{id}` lookup; POSTing /heartbeat would advance the
    //    server's last-seen timestamp as a side effect of running
    //    `doctor`, which is the exact opposite of what a diagnostic
    //    should do.
    if let Ok(ref c) = config_result {
        let client = reqwest::Client::new();
        let url = c.api_url(&format!("/runners/{}", c.runner_id));
        let resp = client
            .get(&url)
            .header(
                AUTHORIZATION,
                HeaderValue::from_str(&c.auth_header()).unwrap(),
            )
            .send()
            .await;
        let token_check = match resp {
            Ok(r) if r.status().is_success() => Ok(format!("OK ({})", r.status())),
            Ok(r) => Err(format!("server returned {}", r.status())),
            Err(e) => Err(format!("{e}")),
        };
        checks.push(("Server reachable + token valid".into(), token_check));
    } else {
        checks.push((
            "Server reachable + token valid".into(),
            Err("skipped (config unreadable)".into()),
        ));
    }
    // 3. Service install record (informational; not a hard requirement)
    let install = runner::service_mode::load(instance);
    checks.push((
        format!("Service install ('{instance}')"),
        match install {
            Some(ref r) => Ok(format!(
                "{} scope, {}",
                r.scope.as_str(),
                r.unit_path.display()
            )),
            None => Ok("not installed (standalone is fine)".into()),
        },
    ));
    // 4. PID file state
    let pid_path = pid_file
        .map(std::path::PathBuf::from)
        .unwrap_or_else(runner::pid_file::default_path);
    let pid_check = match runner::pid_file::inspect(&pid_path) {
        runner::pid_file::Existing::Live(pid) => Ok(format!("running (PID {pid})")),
        runner::pid_file::Existing::Stale(pid) => Err(format!(
            "stale PID {pid} in {}; remove it or run `anvil runner stop`",
            pid_path.display()
        )),
        runner::pid_file::Existing::None => Ok("no PID file (not running standalone)".into()),
    };
    checks.push(("Process state".into(), pid_check));
    // 5. Service-mode consistency: if there's a service install, ensure
    //    its unit file actually exists on disk.
    if let Some(ref r) = install {
        let unit_check = if r.unit_path.exists() {
            Ok(format!("unit file present at {}", r.unit_path.display()))
        } else {
            Err(format!(
                "service.json points at {} but the file is gone — \
                 uninstall and reinstall to fix",
                r.unit_path.display()
            ))
        };
        checks.push(("Service unit on disk".into(), unit_check));
    }
    // Output
    let json_mode = output::is_json();
    if json_mode {
        let arr: Vec<serde_json::Value> = checks
            .iter()
            .map(|(name, result)| {
                let (status, detail) = match result {
                    Ok(s) => ("ok", s),
                    Err(s) => ("error", s),
                };
                serde_json::json!({ "check": name, "status": status, "detail": detail })
            })
            .collect();
        output::print_json(&serde_json::Value::Array(arr));
    } else {
        output::header("Runner Doctor");
        for (name, result) in &checks {
            match result {
                Ok(s) => output::success(&format!("{name}: {s}")),
                Err(s) => output::error(&format!("{name}: {s}")),
            }
        }
    }
    if !config_ok {
        return Err("doctor: configuration is unreadable".into());
    }
    // Surface any Err check as a non-zero exit so CI / monitors that
    // shell out to `anvil runner doctor` can rely on $? to gate.
    // Stale PID, missing unit file, unreachable server — all of these
    // are operator-actionable failures and shouldn't be hidden behind
    // an exit-zero "all good" code path.
    let failed: Vec<&str> = checks
        .iter()
        .filter_map(|(name, r)| r.as_ref().err().map(|_| name.as_str()))
        .collect();
    if !failed.is_empty() {
        return Err(format!("doctor: failed checks: {}", failed.join(", ")).into());
    }
    Ok(())
}
async fn unconfigure(
    config_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -588,18 +1120,75 @@
            scope,
            user_account,
            parallel,
            service_name,
        } => {
            let instance = resolve_instance(service_name)?;
            service_install(InstallOpts {
                config_path: config.as_deref(),
                scope: scope.map(Into::into),
                user_account: user_account.as_deref(),
                parallel,
                instance: &instance,
            })
        }
        ServiceCommand::Uninstall { service_name } => {
            let instance = resolve_instance(service_name)?;
            service_uninstall(&instance)
        }
        ServiceCommand::Start { service_name } => {
            let instance = resolve_instance(service_name)?;
            service_cmd("start", &instance)
        }
        ServiceCommand::Stop { service_name } => {
            let instance = resolve_instance(service_name)?;
            service_cmd("stop", &instance)
        }
        ServiceCommand::Restart { service_name } => {
            let instance = resolve_instance(service_name)?;
            service_cmd("restart", &instance)
        }
        ServiceCommand::SvcStatus { service_name } => {
            let instance = resolve_instance(service_name)?;
            service_cmd("status", &instance)
        }
        ServiceCommand::List => service_list(),
    }
}
/// Validate and unwrap the optional `--service-name`. Empty / missing
/// becomes `DEFAULT_INSTANCE` so the rest of the pipeline can rely on a
/// concrete string.
fn resolve_instance(
    arg: Option<String>,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
    let name = arg.unwrap_or_else(|| runner::service_mode::DEFAULT_INSTANCE.to_string());
    runner::service_mode::validate_instance_name(&name)?;
    Ok(name)
}
fn service_list() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        } => service_install(InstallOpts {
            config_path: config.as_deref(),
            scope: scope.map(Into::into),
            user_account: user_account.as_deref(),
            parallel,
        }),
    let instances = runner::service_mode::list_instances();
    if instances.is_empty() {
        output::info("No runner service instances installed.");
        return Ok(());
        ServiceCommand::Uninstall => service_uninstall(),
        ServiceCommand::Start => service_cmd("start"),
        ServiceCommand::Stop => service_cmd("stop"),
        ServiceCommand::Restart => service_cmd("restart"),
        ServiceCommand::SvcStatus => service_cmd("status"),
    }
    output::header("Installed runner service instances");
    for name in instances {
        if let Some(record) = runner::service_mode::load(&name) {
            output::detail(
                &name,
                &format!(
                    "{} scope, parallel={}, unit={}",
                    record.scope.as_str(),
                    record.parallel,
                    record.unit_path.display()
                ),
            );
        } else {
            output::detail(&name, "<unreadable record>");
        }
    }
    Ok(())
}
struct InstallOpts<'a> {
@@ -607,16 +1196,27 @@
    scope: Option<runner::service_mode::Scope>,
    user_account: Option<&'a str>,
    parallel: Option<u32>,
    instance: &'a str,
}
/// Compute the systemd unit name (Linux) or launchd Label (macOS) for
/// an instance. Both are derived deterministically from the instance
/// name so the operator can grep `systemctl`/`launchctl` output by
/// runner identity.
fn unit_or_label_for(instance: &str) -> String {
    if cfg!(target_os = "macos") {
        format!("com.anvil.runner.{instance}")
    } else {
        format!("anvil-runner-{instance}.service")
    }
}
fn service_install(opts: InstallOpts<'_>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    use runner::service_mode::{InstallRecord, Scope};
    let exe = std::env::current_exe()?;
    let config_file = RunnerConfig::path(opts.config_path);
    // Read parallel from config (if present) so install bakes the right
    // value into ExecStart. Override-able via --parallel.
    let cfg = RunnerConfig::load(opts.config_path).ok();
    let parallel = opts
        .parallel
@@ -636,39 +1236,81 @@
        );
    }
    // Refuse to clobber an existing install at the same instance name.
    if runner::service_mode::load(opts.instance).is_some() {
        return Err(format!(
            "service instance '{}' is already installed; \
             uninstall it first or choose a different --service-name",
            opts.instance
        )
        .into());
    }
    let unit_or_label = unit_or_label_for(opts.instance);
    let unit_path = if cfg!(target_os = "macos") {
        install_launchd(
            &exe,
            &config_file,
            scope,
            opts.user_account,
            parallel,
            opts.instance,
            &unit_or_label,
        )?
        install_launchd(&exe, &config_file, scope, opts.user_account, parallel)?
    } else {
        install_systemd(
            &exe,
            &config_file,
            scope,
            opts.user_account,
            parallel,
            opts.instance,
            &unit_or_label,
        install_systemd(&exe, &config_file, scope, opts.user_account, parallel)?
        )?
    };
    runner::service_mode::save(&InstallRecord {
        instance: opts.instance.to_string(),
        scope,
        unit_path: unit_path.clone(),
        config_path: config_file,
        user_account: opts.user_account.map(String::from),
        parallel,
        unit_or_label,
    })?;
    output::success(&format!(
        "Installed {} service ({} scope): {}",
        "Installed {} service '{}' ({} scope): {}",
        if cfg!(target_os = "macos") {
            "launchd"
        } else {
            "systemd"
        },
        opts.instance,
        scope.as_str(),
        unit_path.display()
    ));
    let start_hint = if opts.instance == runner::service_mode::DEFAULT_INSTANCE {
        "Start with: anvil runner service svc-start".to_string()
    } else {
        format!(
            "Start with: anvil runner service svc-start --service-name {}",
            opts.instance
        )
    };
    output::info("Start with: anvil runner service svc-start");
    output::info(&start_hint);
    Ok(())
}
#[allow(clippy::too_many_arguments)]
fn install_systemd(
    exe: &std::path::Path,
    config_file: &std::path::Path,
    scope: runner::service_mode::Scope,
    user_account: Option<&str>,
    parallel: u32,
    instance: &str,
    unit_name: &str,
) -> Result<std::path::PathBuf, Box<dyn std::error::Error + Send + Sync>> {
    use runner::service_mode::Scope;
@@ -684,42 +1326,65 @@
            dir
        }
    };
    let unit_path = unit_dir.join(unit_name);
    let unit_path = unit_dir.join("anvil-runner.service");
    // Per-instance settings: PID file, working dir, ExecStart flags.
    // Use systemd-managed RuntimeDirectory/StateDirectory for system
    // scope so we don't have to mkdir + chown manually; for user scope
    // we keep state under $HOME via %h expansion.
    let (scope_lines, pid_file_arg, runtime_path_arg) = match scope {
        Scope::System => {
            let u = user_account.unwrap();
            let scope_lines = format!(
                "User={u}\n\
                 Group={u}\n\
                 StateDirectory=anvil-runner-{instance}\n\
                 RuntimeDirectory=anvil-runner-{instance}\n\
                 WorkingDirectory=/var/lib/anvil-runner-{instance}\n\
                 ReadWritePaths=/var/lib/anvil-runner-{instance} /run/anvil-runner-{instance} /tmp\n\
                 ProtectHome=true\n",
            );
            (
                scope_lines,
                format!("/run/anvil-runner-{instance}/runner.pid"),
                format!("/var/lib/anvil-runner-{instance}"),
            )
        }
        Scope::User => {
            // %h expands to the user's home; user units don't read $HOME
            // until ExecStart's argv is parsed but `%h` is a systemd
            // specifier and works in both ExecStart and ReadWritePaths.
            let scope_lines = "WorkingDirectory=%h/.anvil-runner\n\
                 ReadWritePaths=%h/.anvil-runner /tmp\n\
                 ProtectHome=read-only\n"
                .to_string();
            (
                scope_lines,
                format!("%h/.anvil-runner/runner-{instance}.pid"),
                "%h/.anvil-runner".to_string(),
            )
        }
    let user_line = match scope {
        Scope::System => format!(
            "User={}\nGroup={}\n",
            user_account.unwrap(),
            user_account.unwrap()
        ),
        Scope::User => String::new(),
    };
    let _ = runtime_path_arg;
    let work_dir = match scope {
        Scope::System => "WorkingDirectory=/var/lib/anvil-runner\n".to_string(),
        Scope::User => String::new(),
    };
    let unit = format!(
        r#"[Unit]
Description=Anvil CI Runner
Description=Anvil CI Runner ({instance})
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart={exe_str} runner start --config {config_str}
{user_line}{work_dir}Restart=always
ExecStart={exe_str} runner start --config {config_str} --pid-file {pid_file_arg} --parallel {parallel} --shutdown-timeout 60
{scope_lines}Restart=always
RestartSec=5
KillSignal=SIGTERM
TimeoutStopSec=90
StandardOutput=journal
StandardError=journal
ProtectSystem=strict
ProtectHome=true
NoNewPrivileges=true
PrivateTmp=true
ReadWritePaths=/var/lib/anvil-runner /tmp
[Install]
WantedBy={wanted_by}
@@ -730,19 +1395,9 @@
        },
    );
    // ExecStart needs to include --parallel; substitute into the formatted unit.
    // (We pass through `parallel` via the configured config.json as the source
    // of truth, but baking the flag in keeps the unit self-describing.)
    let unit = unit.replace(
        &format!("ExecStart={exe_str} runner start --config {config_str}"),
        &format!("ExecStart={exe_str} runner start --config {config_str} --shutdown-timeout 60"),
    );
    let _ = parallel; // `parallel` is reflected via config.json; --parallel flag override could be added later
    std::fs::write(&unit_path, unit)?;
    let bare_unit = unit_name.trim_end_matches(".service");
    // Reload + enable. Fall through silently on enable failure — the user can
    // run svc-start manually.
    let systemctl_args: Vec<&str> = match scope {
        Scope::User => vec!["--user", "daemon-reload"],
        Scope::System => vec!["daemon-reload"],
@@ -752,8 +1407,8 @@
        .output();
    let enable_args: Vec<&str> = match scope {
        Scope::User => vec!["--user", "enable", bare_unit],
        Scope::User => vec!["--user", "enable", "anvil-runner"],
        Scope::System => vec!["enable", "anvil-runner"],
        Scope::System => vec!["enable", bare_unit],
    };
    let _ = std::process::Command::new("systemctl")
        .args(&enable_args)
@@ -762,33 +1417,47 @@
    Ok(unit_path)
}
#[allow(clippy::too_many_arguments)]
fn install_launchd(
    exe: &std::path::Path,
    config_file: &std::path::Path,
    scope: runner::service_mode::Scope,
    user_account: Option<&str>,
    parallel: u32,
    instance: &str,
    label: &str,
) -> Result<std::path::PathBuf, Box<dyn std::error::Error + Send + Sync>> {
    use runner::service_mode::Scope;
    let _ = parallel; // baked through config.json today; --parallel override is a follow-up
    let exe_str = exe.to_str().unwrap();
    let config_str = config_file.to_str().unwrap();
    let (plist_dir, log_dir, state_dir) = match scope {
        Scope::User => {
            let home = dirs::home_dir().unwrap();
            (
                home.join("Library/LaunchAgents"),
                home.join("Library/Logs/anvil-runner"),
                home.join(".anvil-runner"),
    let (plist_dir, log_dir) = match scope {
        Scope::User => (
            dirs::home_dir().unwrap().join("Library/LaunchAgents"),
            )
        }
            dirs::home_dir().unwrap().join("Library/Logs/anvil-runner"),
        ),
        Scope::System => (
            std::path::PathBuf::from("/Library/LaunchDaemons"),
            std::path::PathBuf::from("/var/log/anvil-runner"),
            std::path::PathBuf::from("/var/lib/anvil-runner"),
        ),
    };
    std::fs::create_dir_all(&plist_dir)?;
    // Log dir failure for system scope is a real error (root needed).
    let _ = std::fs::create_dir_all(&log_dir);
    let plist_path = plist_dir.join("com.anvil.runner.plist");
    if let Err(e) = std::fs::create_dir_all(&log_dir) {
        output::warn(&format!(
            "could not create log dir {}: {e} — service may fail to start",
            log_dir.display()
        ));
    }
    let _ = std::fs::create_dir_all(&state_dir);
    let plist_path = plist_dir.join(format!("{label}.plist"));
    let user_keys = match scope {
        Scope::System => format!(
@@ -798,6 +1467,11 @@
        Scope::User => String::new(),
    };
    let pid_path = state_dir.join(format!("runner-{instance}.pid"));
    let log_path = log_dir.join(format!("{instance}.log"));
    // Single log file: redirect both stdout and stderr to it so a panic
    // landing on stderr shows up in `anvil runner logs`. Otherwise the
    // ".err.log" was a needle in a haystack operators never checked.
    let home = dirs::home_dir().unwrap();
    let plist = format!(
        r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -805,7 +1479,7 @@
<plist version="1.0">
<dict>
    <key>Label</key>
    <string>com.anvil.runner</string>
    <string>{label}</string>
    <key>ProgramArguments</key>
    <array>
        <string>{exe_str}</string>
@@ -813,6 +1487,12 @@
        <string>start</string>
        <string>--config</string>
        <string>{config_str}</string>
        <string>--pid-file</string>
        <string>{pid_str}</string>
        <string>--parallel</string>
        <string>{parallel}</string>
        <string>--shutdown-timeout</string>
        <string>60</string>
    </array>
    <key>RunAtLoad</key>
    <true/>
@@ -825,7 +1505,7 @@
    <key>StandardOutPath</key>
    <string>{log_out}</string>
    <key>StandardErrorPath</key>
    <string>{log_err}</string>
    <string>{log_out}</string>
{user_keys}    <key>EnvironmentVariables</key>
    <dict>
        <key>HOME</key>
@@ -833,7 +1513,7 @@
    </dict>
</dict>
</plist>"#,
        pid_str = pid_path.display(),
        log_out = log_path.display(),
        log_out = log_dir.join("runner.log").display(),
        log_err = log_dir.join("runner.err.log").display(),
        home = home.display(),
    );
@@ -842,22 +1522,28 @@
    Ok(plist_path)
}
fn service_uninstall() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
fn service_uninstall(instance: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    use runner::service_mode::Scope;
    let record = runner::service_mode::load(instance).ok_or_else(|| {
        format!("no service install recorded for '{instance}'; nothing to uninstall")
    })?;
    let record = runner::service_mode::load()
        .ok_or_else(|| "no service install recorded; nothing to uninstall".to_string())?;
    if cfg!(target_os = "macos") {
        // Best-effort unload first (so the running daemon goes away
        // before we delete the plist).
        let _ = std::process::Command::new("launchctl")
        // Modern: bootout. Older macOS falls back to unload. We log a
        // warning on failure but proceed with file removal: a failed
        // bootout on uninstall typically means "not currently loaded",
        // which is the desired end state anyway. Surfacing the message
        // so the operator can investigate if it's actually a permission
        // issue rather than a "not loaded" no-op.
        if let Err(e) = launchctl_bootout(&record) {
            output::warn(&format!("launchctl bootout: {e}"));
        }
            .args(["unload", record.unit_path.to_str().unwrap()])
            .output();
    } else {
        let bare_unit = record.unit_or_label.trim_end_matches(".service");
        let args: Vec<&str> = match record.scope {
            Scope::User => vec!["--user", "disable", bare_unit],
            Scope::System => vec!["disable", bare_unit],
            Scope::User => vec!["--user", "disable", "anvil-runner"],
            Scope::System => vec!["disable", "anvil-runner"],
        };
        let _ = std::process::Command::new("systemctl").args(&args).output();
    }
@@ -865,74 +1551,58 @@
    if record.unit_path.exists() {
        std::fs::remove_file(&record.unit_path)?;
    }
    runner::service_mode::clear()?;
    runner::service_mode::clear(instance)?;
    output::success(&format!(
        "Uninstalled {} service ({} scope)",
        "Uninstalled {} service '{}' ({} scope)",
        if cfg!(target_os = "macos") {
            "launchd"
        } else {
            "systemd"
        },
        instance,
        record.scope.as_str()
    ));
    Ok(())
}
fn service_cmd(action: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
fn service_cmd(
    action: &str,
    instance: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    use runner::service_mode::Scope;
    let record = runner::service_mode::load(instance).ok_or_else(|| {
        format!(
    let record = runner::service_mode::load().ok_or_else(|| {
        "no service install recorded; run `anvil runner service install` first".to_string()
            "no service install recorded for '{instance}'; \
             run `anvil runner service install` first"
        )
    })?;
    if cfg!(target_os = "macos") {
        match action {
            "start" => launchctl_bootstrap(&record)?,
            "stop" => launchctl_bootout(&record)?,
            "restart" => {
        let plist_str = record.unit_path.to_str().unwrap();
        // launchctl has no restart verb; emulate with unload + load.
        // Unload failures during restart are non-fatal (service may
        // already be stopped).
        let launchctl_actions: &[&str] = match action {
            "start" => &["load"],
            "stop" => &["unload"],
            "restart" => &["unload", "load"],
                // Best-effort bootout (may not be loaded yet), then bootstrap.
                let _ = launchctl_bootout(&record);
                launchctl_bootstrap(&record)?;
            }
            "status" => {
                let output = std::process::Command::new("launchctl")
                    .args(["list", "com.anvil.runner"])
                    .output()?;
                println!("{}", String::from_utf8_lossy(&output.stdout));
                launchctl_print(&record)?;
                return Ok(());
            }
            _ => return Err(format!("unknown action: {action}").into()),
        };
        let mut last_failure: Option<String> = None;
        for (idx, launchctl_action) in launchctl_actions.iter().enumerate() {
            let output = std::process::Command::new("launchctl")
                .args([launchctl_action, plist_str])
                .output()?;
            if !output.status.success() {
                let stderr = String::from_utf8_lossy(&output.stderr).to_string();
                let is_first_of_pair = action == "restart" && idx == 0;
                if is_first_of_pair {
                    continue;
                }
                last_failure = Some(format!("launchctl {launchctl_action} failed: {stderr}"));
                break;
            }
        }
        if let Some(err) = last_failure {
            return Err(err.into());
        }
        output::success(&format!("Service {action}ed"));
    } else {
        let bare_unit = record.unit_or_label.trim_end_matches(".service");
        let mut args: Vec<&str> = Vec::new();
        if matches!(record.scope, Scope::User) {
            args.push("--user");
        }
        args.push(action);
        args.push("anvil-runner");
        args.push(bare_unit);
        let output = std::process::Command::new("systemctl")
            .args(&args)
@@ -950,6 +1620,171 @@
    Ok(())
}
// ── macOS modern launchctl (bootstrap/bootout/print) ────────────────
#[cfg(target_os = "macos")]
fn launchctl_domain(record: &runner::service_mode::InstallRecord) -> String {
    match record.scope {
        runner::service_mode::Scope::User => {
            let uid = unsafe { libc::getuid() };
            format!("gui/{uid}")
        }
        runner::service_mode::Scope::System => "system".to_string(),
    }
}
#[cfg(target_os = "macos")]
fn launchctl_run(args: &[&str]) -> std::io::Result<std::process::Output> {
    std::process::Command::new("launchctl").args(args).output()
}
#[cfg(target_os = "macos")]
fn launchctl_bootstrap(
    record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let plist = record.unit_path.to_string_lossy().to_string();
    let primary_domain = launchctl_domain(record);
    let try_bootstrap = |domain: &str| -> std::io::Result<std::process::Output> {
        launchctl_run(&["bootstrap", domain, &plist])
    };
    let out = try_bootstrap(&primary_domain)?;
    if out.status.success() {
        return Ok(());
    }
    let stderr = String::from_utf8_lossy(&out.stderr).to_lowercase();
    // Already-loaded — treat as success.
    // launchctl emits a few different phrasings: "already bootstrapped",
    // "service is already loaded", "Service already loaded", and EEXIST
    // (exit code 17) when the service is in the domain. Cover all of
    // them lowercased so casing variance can't break idempotence.
    let already_loaded = stderr.contains("already")
        || stderr.contains("service already")
        || out.status.code() == Some(17);
    if already_loaded {
        return Ok(());
    }
    // Old macOS: fall back to deprecated `load -w`.
    if stderr.contains("unrecognized subcommand") {
        let fallback = launchctl_run(&["load", "-w", &plist])?;
        if !fallback.status.success() {
            return Err(format!(
                "launchctl load failed: {}",
                String::from_utf8_lossy(&fallback.stderr)
            )
            .into());
        }
        return Ok(());
    }
    // Headless / ssh sessions have no `gui/<uid>` domain because there's
    // no Aqua session for that user. launchctl reports something like
    // "Bootstrap failed: 5: Input/output error" or "Could not find
    // domain". Retry with the `user/<uid>` domain, which exists as soon
    // as the user has any process running (i.e., ssh shell is enough).
    if matches!(record.scope, runner::service_mode::Scope::User)
        && primary_domain.starts_with("gui/")
        && (stderr.contains("could not find domain")
            || stderr.contains("domain does not")
            || stderr.contains("input/output error"))
    {
        let uid = unsafe { libc::getuid() };
        let user_domain = format!("user/{uid}");
        let out2 = try_bootstrap(&user_domain)?;
        if out2.status.success() {
            return Ok(());
        }
        let stderr2 = String::from_utf8_lossy(&out2.stderr).to_lowercase();
        if stderr2.contains("already") || out2.status.code() == Some(17) {
            return Ok(());
        }
        return Err(format!(
            "launchctl bootstrap failed in both {primary_domain} and {user_domain}: {stderr2}"
        )
        .into());
    }
    Err(format!("launchctl bootstrap failed: {stderr}").into())
}
#[cfg(target_os = "macos")]
fn launchctl_bootout(
    record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let domain = launchctl_domain(record);
    let target = format!("{domain}/{}", record.unit_or_label);
    let out = launchctl_run(&["bootout", &target])?;
    if out.status.success() {
        return Ok(());
    }
    let stderr = String::from_utf8_lossy(&out.stderr).to_lowercase();
    // Not-found / not-loaded → treat as success (nothing to stop).
    // launchctl returns ESRCH (3) or ENOENT (2) for these cases; we
    // lowercase the stderr to absorb casing differences across macOS
    // versions.
    if stderr.contains("not find")
        || stderr.contains("no such")
        || stderr.contains("could not find")
        || matches!(out.status.code(), Some(2) | Some(3))
    {
        return Ok(());
    }
    // Old macOS: fall back to `unload`.
    if stderr.contains("unrecognized subcommand") {
        let fallback = launchctl_run(&["unload", &record.unit_path.to_string_lossy()])?;
        if !fallback.status.success() {
            return Err(format!(
                "launchctl unload failed: {}",
                String::from_utf8_lossy(&fallback.stderr)
            )
            .into());
        }
        return Ok(());
    }
    Err(format!("launchctl bootout failed: {stderr}").into())
}
#[cfg(target_os = "macos")]
fn launchctl_print(
    record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let domain = launchctl_domain(record);
    let target = format!("{domain}/{}", record.unit_or_label);
    let out = launchctl_run(&["print", &target])?;
    if out.status.success() {
        println!("{}", String::from_utf8_lossy(&out.stdout));
        return Ok(());
    }
    let stderr = String::from_utf8_lossy(&out.stderr).to_lowercase();
    if stderr.contains("unrecognized subcommand") {
        // Fall back to deprecated `list`.
        let fallback = launchctl_run(&["list", &record.unit_or_label])?;
        println!("{}", String::from_utf8_lossy(&fallback.stdout));
        return Ok(());
    }
    Err(format!("launchctl print failed: {stderr}").into())
}
#[cfg(not(target_os = "macos"))]
fn launchctl_bootout(
    _record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    Err("launchctl unavailable on this platform".into())
}
#[cfg(not(target_os = "macos"))]
fn launchctl_bootstrap(
    _record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    Err("launchctl unavailable on this platform".into())
}
#[cfg(not(target_os = "macos"))]
fn launchctl_print(
    _record: &runner::service_mode::InstallRecord,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    Err("launchctl unavailable on this platform".into())
}
#[cfg(unix)]
fn unix_is_root() -> bool {
    (unsafe { libc::getuid() }) == 0
@@ -1161,6 +1996,6 @@
    fn svc_start_parses_to_start_variant() {
        assert!(matches!(
            parse(&["service", "svc-start"]),
            ServiceCommand::Start { .. }
            ServiceCommand::Start
        ));
    }
@@ -1169,6 +2004,6 @@
    fn svc_stop_parses_to_stop_variant() {
        assert!(matches!(
            parse(&["service", "svc-stop"]),
            ServiceCommand::Stop
            ServiceCommand::Stop { .. }
        ));
    }
@@ -1177,6 +2012,6 @@
    fn svc_restart_parses_to_restart_variant() {
        assert!(matches!(
            parse(&["service", "svc-restart"]),
            ServiceCommand::Restart
            ServiceCommand::Restart { .. }
        ));
    }
@@ -1185,10 +2020,25 @@
    fn svc_status_parses_to_status_variant() {
        assert!(matches!(
            parse(&["service", "svc-status"]),
            ServiceCommand::SvcStatus { .. }
            ServiceCommand::SvcStatus
        ));
    }
    #[test]
    fn svc_start_parses_service_name_flag() {
        match parse(&["service", "svc-start", "--service-name", "gpu1"]) {
            ServiceCommand::Start { service_name } => {
                assert_eq!(service_name.as_deref(), Some("gpu1"));
            }
            _ => panic!("expected Start"),
        }
    }
    #[test]
    fn service_list_parses() {
        assert!(matches!(parse(&["service", "list"]), ServiceCommand::List));
    }
    fn parse_runner(args: &[&str]) -> RunnerCommand {
        RunnerCli::try_parse_from(args).expect("parse").command
    }
@@ -1200,6 +2050,7 @@
                pid_file,
                timeout,
                force,
                ..
            } => {
                assert!(pid_file.is_none());
                assert!(timeout >= 60);
@@ -1288,13 +2139,89 @@
                scope,
                user_account,
                parallel,
                service_name,
                ..
            } => {
                assert!(scope.is_none());
                assert!(user_account.is_none());
                assert!(parallel.is_none());
                assert!(service_name.is_none());
            }
            _ => panic!("expected Install"),
        }
    }
    #[test]
    fn service_install_accepts_service_name() {
        match parse(&["service", "install", "--service-name", "gpu1"]) {
            ServiceCommand::Install { service_name, .. } => {
                assert_eq!(service_name.as_deref(), Some("gpu1"));
            }
            _ => panic!("expected Install"),
        }
    }
    #[test]
    fn start_parses_detach_and_log_file() {
        match parse_runner(&["start", "--detach", "--log-file", "/var/log/anvil.log"]) {
            RunnerCommand::Start {
                detach,
                log_file,
                detach_child,
                ..
            } => {
                assert!(detach);
                assert!(!detach_child);
                assert_eq!(log_file.as_deref(), Some("/var/log/anvil.log"));
            }
            _ => panic!("expected Start"),
        }
    }
    #[test]
    fn start_parses_hidden_detach_child_flag() {
        match parse_runner(&["start", "--detach-child"]) {
            RunnerCommand::Start { detach_child, .. } => {
                assert!(detach_child);
            }
            _ => panic!("expected Start"),
        }
    }
    #[test]
    fn status_parses_with_service_name() {
        match parse_runner(&["status", "--service-name", "gpu1"]) {
            RunnerCommand::Status { service_name, .. } => {
                assert_eq!(service_name.as_deref(), Some("gpu1"));
            }
            _ => panic!("expected Status"),
        }
    }
    #[test]
    fn logs_parses_follow_and_lines() {
        match parse_runner(&["logs", "-f", "-n", "500"]) {
            RunnerCommand::Logs { follow, lines, .. } => {
                assert!(follow);
                assert_eq!(lines, 500);
            }
            _ => panic!("expected Logs"),
        }
    }
    #[test]
    fn doctor_parses_with_defaults() {
        match parse_runner(&["doctor"]) {
            RunnerCommand::Doctor {
                config,
                pid_file,
                service_name,
            } => {
                assert!(config.is_none());
                assert!(pid_file.is_none());
                assert!(service_name.is_none());
            }
            _ => panic!("expected Doctor"),
        }
    }
}

@@ -1,0 +1,179 @@
//! Detach a `runner start` invocation into the background.
//!
//! We avoid the classic double-fork dance — by the time we reach this
//! point we're already inside a tokio runtime, and `fork()` while tokio
//! has spawned worker threads is undefined behavior. Instead we re-exec
//! ourselves with a hidden `--detach-child` flag, redirecting the
//! child's stdio to a rotated log file and placing it in a new process
//! group so terminal-close signals don't reach it. The parent then exits
//! with the child PID printed for the operator.
//!
//! The child runs the normal `runner start` path; the `--detach-child`
//! flag exists only to prevent infinite re-detach recursion if the
//! daemon ever re-invokes itself.
use crate::runner::log_rotate::{self, RotationPolicy};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
/// Default location for the detached daemon's stdout/stderr log.
pub fn default_log_path() -> PathBuf {
    dirs::home_dir()
        .unwrap_or_else(|| PathBuf::from("."))
        .join(".anvil-runner")
        .join("runner.log")
}
/// Spawn the current process tree as a detached daemon, redirecting
/// stdout+stderr to `log_path` and giving it a new process group.
/// Returns the spawned child's PID. The caller is expected to exit
/// after this returns.
///
/// `forwarded_args` are the original `runner start ...` flags minus
/// `--detach` (so the child doesn't try to detach again). Caller adds
/// the `--detach-child` hidden flag as a marker.
pub fn spawn_daemon(
    log_path: &Path,
    forwarded_args: &[String],
) -> Result<u32, Box<dyn std::error::Error + Send + Sync>> {
    // Rotate the existing log if it's grown beyond the threshold. This
    // bounds disk usage across restarts; long-running detached runners
    // that need stricter rotation should pair with logrotate / newsyslog.
    let _ = log_rotate::maybe_rotate(log_path, RotationPolicy::default())?;
    if let Some(parent) = log_path.parent() {
        std::fs::create_dir_all(parent)?;
    }
    // Open the log file once and dup it to both stdout and stderr so the
    // ordering of stdout/stderr writes from the daemon matches the wall
    // clock. APPEND so concurrent runners (e.g. user accidentally starts
    // two) don't clobber each other's tails.
    let log_file = std::fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(log_path)?;
    let log_clone = log_file.try_clone()?;
    let exe = std::env::current_exe()?;
    let mut cmd = std::process::Command::new(exe);
    cmd.arg("runner")
        .arg("start")
        .arg("--detach-child")
        .args(forwarded_args)
        .stdin(std::process::Stdio::null())
        .stdout(log_file)
        .stderr(log_clone);
    #[cfg(unix)]
    {
        use std::os::unix::process::CommandExt;
        cmd.process_group(0);
        // setsid() detaches the child from the parent's terminal session
        // BEFORE our tokio SIGHUP handler arms; without this, a shell
        // with `huponexit` set can kill the daemon during its startup
        // window. pre_exec runs in the forked child after fork() but
        // before execvp(), so it's safe to call libc::setsid directly.
        unsafe {
            cmd.pre_exec(|| {
                // setsid creates a new session AND process group, making
                // the child the leader of both. Best-effort: if the
                // child is already a process group leader,
                // process_group(0) above already covered that.
                if libc::setsid() == -1 {
                    let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
                    // EPERM means the caller is already a session leader
                    // (extremely unlikely in our spawn path, but harmless).
                    if errno != libc::EPERM {
                        return Err(std::io::Error::last_os_error());
                    }
                }
                Ok(())
            });
        }
    }
    let child = cmd.spawn()?;
    Ok(child.id())
}
/// Outcome of waiting for the detached daemon to come up.
pub enum DaemonReady {
    /// Daemon wrote its PID file; the daemon's real PID is enclosed
    /// (typically the same as the spawn'd PID unless the daemon
    /// double-forks itself in the future).
    Live(u32),
    /// Daemon process exited during startup. The string is the tail of
    /// its log file for the operator to triage.
    DiedDuringStartup(String),
    /// Daemon process is still alive but hasn't acquired its PID file
    /// within the startup window; could be a slow heartbeat call.
    Timeout,
}
const STARTUP_TIMEOUT_SECS: u64 = 10;
const POLL_INTERVAL_MS: u64 = 100;
/// Block the parent until the spawned daemon either:
///   1. acquires its PID file (success — return Live with the PID
///      read from the file, which may differ from spawn_pid if the
///      daemon double-forks later),
///   2. dies entirely (return DiedDuringStartup with the log tail), or
///   3. exhausts the startup budget (return Timeout — caller decides
///      whether to fail or warn).
pub fn wait_for_daemon_ready(
    spawn_pid: u32,
    pid_path: &Path,
    log_path: &Path,
) -> std::io::Result<DaemonReady> {
    let deadline = Instant::now() + Duration::from_secs(STARTUP_TIMEOUT_SECS);
    let interval = Duration::from_millis(POLL_INTERVAL_MS);
    while Instant::now() < deadline {
        // (1) PID file present?
        if let Ok(contents) = std::fs::read_to_string(pid_path) {
            if let Ok(pid) = contents.trim().parse::<u32>() {
                return Ok(DaemonReady::Live(pid));
            }
        }
        // (2) Did the spawned process die?
        #[cfg(unix)]
        {
            let alive = unsafe { libc::kill(spawn_pid as i32, 0) } == 0;
            if !alive {
                let tail = log_tail(log_path).unwrap_or_else(|| "(log unavailable)".into());
                return Ok(DaemonReady::DiedDuringStartup(tail));
            }
        }
        std::thread::sleep(interval);
    }
    Ok(DaemonReady::Timeout)
}
fn log_tail(path: &Path) -> Option<String> {
    use std::io::Read;
    let mut file = std::fs::File::open(path).ok()?;
    let len = file.metadata().ok()?.len();
    // Read the last ~4 KB so we capture a reasonable amount of context
    // without slurping a huge log file into memory.
    let chunk = 4096u64.min(len);
    let seek_from = std::io::SeekFrom::End(-(chunk as i64));
    use std::io::Seek;
    file.seek(seek_from).ok()?;
    let mut buf = Vec::new();
    file.read_to_end(&mut buf).ok()?;
    let s = String::from_utf8_lossy(&buf).into_owned();
    let tail: String = s
        .lines()
        .rev()
        .take(20)
        .collect::<Vec<_>>()
        .into_iter()
        .rev()
        .collect::<Vec<_>>()
        .join("\n");
    Some(tail)
}

@@ -10,8 +10,43 @@
/// How long a SIGTERM'd child gets to exit cleanly before we send SIGKILL.
const CHILD_TERM_GRACE_SECS: u64 = 5;
/// Why a job ended, distinguishing runner-initiated cancellation (which
/// the server cares about — those jobs aren't real failures) from a
/// normal exit. Returned alongside the exit code so `report_result` can
/// branch without resorting to brittle exit-code conventions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EndReason {
    /// Child process finished on its own; exit code is the truth.
    Finished,
    /// Runner-wide shutdown signal fired mid-job; child got SIGTERM/SIGKILL.
    RunnerShutdown,
    /// Per-job timeout (timeout_seconds) elapsed; child got SIGKILL.
    Timeout,
}
impl EndReason {
    /// Field value for the job-status update payload's `cancel_reason`.
    /// `None` for `Finished` so the field is omitted entirely.
    pub fn as_cancel_reason(self) -> Option<&'static str> {
        match self {
            EndReason::Finished => None,
            EndReason::RunnerShutdown => Some("runner_shutdown"),
            EndReason::Timeout => Some("timeout"),
        }
    }
    /// `true` if this should be reported as `cancelled` rather than
    /// `passed`/`failed`. Runner-shutdown is the only true cancel —
    /// timeouts are still job-level failures from the server's
    /// perspective.
    pub fn is_runner_cancel(self) -> bool {
        matches!(self, EndReason::RunnerShutdown)
    }
}
pub struct ExecResult {
    pub exit_code: i32,
    pub end_reason: EndReason,
}
/// Execute a job command, streaming output to the log reporter.
@@ -327,20 +362,20 @@
        },
    };
    let exit_code = match outcome {
    let (exit_code, end_reason) = match outcome {
        Outcome::Cancelled => {
            log_reporter
                .append("Runner shutdown — terminating job")
                .await;
            terminate_child(&mut child, log_reporter).await;
            (130, EndReason::RunnerShutdown)
            130
        }
        Outcome::TimedOut => {
            let _ = child.kill().await;
            log_reporter.append("Job timed out, killed").await;
            1
            (1, EndReason::Timeout)
        }
        Outcome::Finished(Ok(status)) => (status.code().unwrap_or(1), EndReason::Finished),
        Outcome::Finished(Ok(status)) => status.code().unwrap_or(1),
        Outcome::Finished(Err(e)) => {
            let _ = stdout_task.await;
            let _ = stderr_task.await;
@@ -354,7 +389,10 @@
    let _ = stderr_task.await;
    log_reporter.flush().await;
    Ok(ExecResult { exit_code })
    Ok(ExecResult {
        exit_code,
        end_reason,
    })
}
/// Send SIGTERM to a running child, wait briefly, then SIGKILL if still
@@ -377,5 +415,37 @@
            ))
            .await;
        let _ = child.kill().await;
    }
}
#[cfg(test)]
mod end_reason_tests {
    use super::EndReason;
    #[test]
    fn finished_has_no_cancel_reason() {
        assert!(EndReason::Finished.as_cancel_reason().is_none());
        assert!(!EndReason::Finished.is_runner_cancel());
    }
    #[test]
    fn runner_shutdown_reports_as_cancel() {
        // The whole point of the cancel_reason field: server's
        // distinguishing a runner-yanked job from a failed/timed-out
        // one. If this regresses, server-side cancel handling breaks.
        assert_eq!(
            EndReason::RunnerShutdown.as_cancel_reason(),
            Some("runner_shutdown")
        );
        assert!(EndReason::RunnerShutdown.is_runner_cancel());
    }
    #[test]
    fn timeout_has_reason_but_is_not_runner_cancel() {
        // Timeout produces a reason string ("timeout") but the job is
        // reported as `failed`, not `cancelled` — so loop_runner only
        // attaches cancel_reason when is_runner_cancel() is true.
        assert_eq!(EndReason::Timeout.as_cancel_reason(), Some("timeout"));
        assert!(!EndReason::Timeout.is_runner_cancel());
    }
}

@@ -1,0 +1,174 @@
//! Tiny size-based log rotation. No external deps. Used by `--detach`
//! when the runner redirects its stdout/stderr to a file.
//!
//! Rotation is rotate-on-start: when `runner start --detach` is invoked,
//! we check the log size and, if over the configured threshold, shuffle
//! `runner.log` → `runner.log.1`, the old `.1` → `.2`, etc., dropping
//! anything past `max_files`. Then we open `runner.log` fresh for the
//! new daemon.
//!
//! This is deliberately not periodic-during-run: a long-lived detached
//! runner that needs strict rotation should pair with `logrotate` (Linux)
//! or `newsyslog` (macOS). The rotate-on-start approach bounds disk
//! usage across restarts, which is the common case (binary upgrades via
//! `runner restart` reset the log naturally).
use std::fs;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Copy)]
pub struct RotationPolicy {
    pub max_size_bytes: u64,
    pub max_files: usize,
}
impl Default for RotationPolicy {
    fn default() -> Self {
        Self {
            // 100 MB per file × 3 rotated copies = 400 MB total cap.
            max_size_bytes: 100 * 1024 * 1024,
            max_files: 3,
        }
    }
}
/// If `path` exists and exceeds `policy.max_size_bytes`, shift the
/// rotation chain (`runner.log.{n-1} → runner.log.n`, … `runner.log →
/// runner.log.1`) and drop anything past `policy.max_files`. The
/// caller then opens a fresh `path` for writing.
///
/// Returns `Ok(true)` if a rotation actually happened, `Ok(false)` if
/// no rotation was needed (file absent or under the threshold).
pub fn maybe_rotate(path: &Path, policy: RotationPolicy) -> std::io::Result<bool> {
    let size = match fs::metadata(path) {
        Ok(m) => m.len(),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
        Err(e) => return Err(e),
    };
    if size <= policy.max_size_bytes {
        return Ok(false);
    }
    rotate_chain(path, policy.max_files)?;
    Ok(true)
}
fn rotated_path(base: &Path, n: usize) -> PathBuf {
    let mut s = base.as_os_str().to_owned();
    s.push(format!(".{n}"));
    PathBuf::from(s)
}
fn rotate_chain(base: &Path, max_files: usize) -> std::io::Result<()> {
    // Drop the oldest copy if it exists. `max_files = 3` means we keep
    // .1, .2, .3 and delete anything that would have been .4.
    let oldest = rotated_path(base, max_files);
    if oldest.exists() {
        fs::remove_file(&oldest)?;
    }
    // Shuffle: .{n-1} → .{n} for n in (max_files .. 2)
    for n in (2..=max_files).rev() {
        let from = rotated_path(base, n - 1);
        let to = rotated_path(base, n);
        if from.exists() {
            fs::rename(&from, &to)?;
        }
    }
    // base → .1
    fs::rename(base, rotated_path(base, 1))?;
    Ok(())
}
#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write;
    fn temp_dir(tag: &str) -> PathBuf {
        let dir =
            std::env::temp_dir().join(format!("anvil-runner-rotate-{}-{tag}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        dir
    }
    fn write_n_bytes(p: &Path, n: usize) {
        let mut f = fs::File::create(p).unwrap();
        f.write_all(&vec![b'x'; n]).unwrap();
    }
    #[test]
    fn no_rotate_when_under_threshold() {
        let dir = temp_dir("under");
        let log = dir.join("runner.log");
        write_n_bytes(&log, 100);
        let rotated = maybe_rotate(
            &log,
            RotationPolicy {
                max_size_bytes: 1000,
                max_files: 3,
            },
        )
        .unwrap();
        assert!(!rotated);
        assert_eq!(fs::metadata(&log).unwrap().len(), 100);
    }
    #[test]
    fn rotates_when_over_threshold() {
        let dir = temp_dir("over");
        let log = dir.join("runner.log");
        write_n_bytes(&log, 2000);
        let rotated = maybe_rotate(
            &log,
            RotationPolicy {
                max_size_bytes: 1000,
                max_files: 3,
            },
        )
        .unwrap();
        assert!(rotated);
        assert!(!log.exists(), "original should be moved aside");
        assert!(
            rotated_path(&log, 1).exists(),
            ".1 should now hold prior contents"
        );
    }
    #[test]
    fn rotation_chain_shifts_and_drops_oldest() {
        let dir = temp_dir("chain");
        let log = dir.join("runner.log");
        // Pre-populate .1, .2, .3 with distinguishable contents.
        fs::write(rotated_path(&log, 1), b"one").unwrap();
        fs::write(rotated_path(&log, 2), b"two").unwrap();
        fs::write(rotated_path(&log, 3), b"three").unwrap();
        write_n_bytes(&log, 5000);
        maybe_rotate(
            &log,
            RotationPolicy {
                max_size_bytes: 1000,
                max_files: 3,
            },
        )
        .unwrap();
        // .3 should now hold what was .2 (the oldest, "three", got dropped).
        assert_eq!(fs::read_to_string(rotated_path(&log, 3)).unwrap(), "two");
        assert_eq!(fs::read_to_string(rotated_path(&log, 2)).unwrap(), "one");
        // .1 holds the prior live log (5000 bytes of 'x').
        assert_eq!(fs::metadata(rotated_path(&log, 1)).unwrap().len(), 5000);
    }
    #[test]
    fn no_rotate_when_log_absent() {
        let dir = temp_dir("absent");
        let log = dir.join("never-existed.log");
        let rotated = maybe_rotate(&log, RotationPolicy::default()).unwrap();
        assert!(!rotated);
    }
}