ref:main
//! Adversarial contract fuzzer: the ONE invariant, checked against EVERY leaf.
//!
//! Under `--json`, stdout must be exactly one valid JSON value — the success
//! payload, or a `{"ok":false,"error":…}` envelope on failure — for every leaf
//! command, whatever the server does. This test walks the real clap tree,
//! synthesizes just-enough arguments for each leaf, and runs the built binary
//! with `--json` against a catch-all mock that answers every request with an
//! adversarial body (an object where several commands expect a bare array, and
//! vice-versa). It then asserts stdout parses as JSON.
//!
//! What this catches that the happy-path tests can't:
//! * a raw `println!`/`print!` that leaked human text onto stdout,
//! * a command that returns without emitting (empty stdout),
//! * a panic on an unexpected server shape (backtrace, partial stdout),
//! * a gate/exit path that forgets to emit its JSON first.
//!
//! Commands that hang (long-poll / exec), mutate the working tree (git), or
//! replace the binary are on an explicit denylist, logged so coverage is honest.
use clap::CommandFactory;
use serde_json::json;
use std::io::Write;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use wiremock::matchers::any;
use wiremock::{Mock, MockServer, ResponseTemplate};
/// Leaves we can't safely spawn in a loop: they block forever, mutate the
/// checkout, touch the real system, or replace the running binary. Each is
/// covered (or excluded by nature) elsewhere; listed here so the skip is loud.
const DENY: &[&str] = &[
// Long-poll / never-returning / re-exec:
"runner start",
"runner restart",
"auth login",
// Writes local runner config / deregisters / system services:
"runner configure",
"runner unconfigure",
"runner logs",
"runner service install",
"runner service uninstall",
"runner service start",
"runner service stop",
"runner service restart",
"runner service status",
"runner service list",
// Mutates the git working tree / clones:
"pr checkout",
"pr diff",
"repo clone",
// Writes files into the cwd / replaces the binary:
"requirement export",
"requirement seed",
"release download",
"update",
];
/// A catch-all body deliberately shaped to trip lazy assumptions: it is an
/// object, but also carries arrays under the common envelope keys, a plaintext
/// `token`, and an `id` — so a command that blindly indexes, unwraps, or expects
/// the opposite container still has to produce *valid JSON* (payload or error).
fn adversarial_body() -> serde_json::Value {
json!({
"id": "11111111-1111-1111-1111-111111111111",
"token": "anvil_secret_plaintext",
"name": "x",
"status": "running",
"data": [],
"items": [],
"requirements": [],
"standards": [],
"columns": [],
"deployments": [],
"environments": [],
"releases": [],
"runner": { "id": "x", "name": "x", "status": "online" },
"label": { "id": "x", "name": "x" },
"milestone": { "id": "x", "title": "x" },
"column": { "id": "x", "name": "x" },
"release": { "id": "x", "tag_name": "x" }
})
}
/// Collect every leaf as `(space-joined path, argv-after-the-path)`, where argv
/// is the minimal set of required args clap needs to parse the leaf.
fn leaves_with_args() -> Vec<(String, Vec<String>)> {
let root = anvil::commands::Cli::command();
let mut out = Vec::new();
walk(&root, Vec::new(), &mut out);
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
fn walk(cmd: &clap::Command, path: Vec<String>, out: &mut Vec<(String, Vec<String>)>) {
let subs: Vec<&clap::Command> = cmd.get_subcommands().collect();
if subs.is_empty() {
if !path.is_empty() {
out.push((path.join(" "), synth_args(cmd)));
}
return;
}
for sub in subs {
let mut p = path.clone();
p.push(sub.get_name().to_string());
walk(sub, p, out);
}
}
/// Synthesize the required arguments for a leaf: a value per required option and
/// positional. Enum args get their first legal variant; everything else a
/// generic token (wrong values are fine — an error still yields a JSON envelope).
fn synth_args(leaf: &clap::Command) -> Vec<String> {
let mut opts: Vec<String> = Vec::new();
let mut positionals: Vec<String> = Vec::new();
for arg in leaf.get_arguments() {
if !arg.is_required_set() {
continue;
}
// "1" parses as both an integer arg (<NUMBER>, <EPIC>) and a string; a
// wrong-but-parseable value is fine, since a runtime error still yields a
// JSON envelope. Enum args get their first legal variant instead.
let value = arg
.get_possible_values()
.first()
.map(|p| p.get_name().to_string())
.unwrap_or_else(|| "1".to_string());
if arg.is_positional() {
positionals.push(value);
} else if let Some(long) = arg.get_long() {
opts.push(format!("--{long}"));
if takes_value(arg) {
opts.push(value);
}
}
}
opts.extend(positionals);
opts
}
fn takes_value(arg: &clap::Arg) -> bool {
matches!(
arg.get_action(),
clap::ArgAction::Set | clap::ArgAction::Append
)
}
/// Run one leaf with `--json` against the mock, killing it after a deadline so a
/// missed hang surfaces as a failure instead of wedging the suite.
fn run_leaf(server_uri: &str, config_path: &str, path: &str, args: &[String]) -> LeafOutcome {
let mut full: Vec<String> = path.split(' ').map(String::from).collect();
full.push("--json".into());
full.extend(args.iter().cloned());
let mut child = Command::new(env!("CARGO_BIN_EXE_anvil"))
.args(&full)
.env("ANVIL_SERVER_URL", server_uri)
.env("ANVIL_TOKEN", "test-token")
.env("ANVIL_CONFIG", config_path)
.env_remove("ANVIL_RUNNER_TOKEN")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn anvil");
let deadline = Instant::now() + Duration::from_secs(20);
loop {
if let Some(_status) = child.try_wait().expect("try_wait") {
let out = child.wait_with_output().expect("wait_with_output");
return LeafOutcome::Exited {
stdout: out.stdout,
stderr: out.stderr,
};
}
if Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
return LeafOutcome::TimedOut;
}
std::thread::sleep(Duration::from_millis(25));
}
}
enum LeafOutcome {
Exited { stdout: Vec<u8>, stderr: Vec<u8> },
TimedOut,
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn every_leaf_emits_valid_json_under_json() {
let server = MockServer::start().await;
Mock::given(any())
.respond_with(ResponseTemplate::new(200).set_body_json(adversarial_body()))
.mount(&server)
.await;
let uri = server.uri();
// Scratch config so repo resolution finds a default and never the dev's real
// config or the checkout's git remote.
let dir = std::env::temp_dir().join(format!("anvil-fuzz-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let config_path = dir.join("config.json");
let mut f = std::fs::File::create(&config_path).unwrap();
f.write_all(
serde_json::to_string_pretty(&json!({
"server_url": uri,
"token": "test-token",
"default_repo": "test-org/test-repo"
}))
.unwrap()
.as_bytes(),
)
.unwrap();
let config_path = config_path.to_str().unwrap().to_string();
let mut failures: Vec<String> = Vec::new();
let mut skipped: Vec<String> = Vec::new();
let mut checked = 0usize;
for (path, args) in leaves_with_args() {
if DENY.contains(&path.as_str()) {
skipped.push(path);
continue;
}
checked += 1;
match run_leaf(&uri, &config_path, &path, &args) {
LeafOutcome::TimedOut => {
failures.push(format!(
"`{path}` [{}]: TIMED OUT (hang — fix or denylist)",
args.join(" ")
));
}
LeafOutcome::Exited { stdout, stderr } => {
if serde_json::from_slice::<serde_json::Value>(&stdout).is_err() {
failures.push(format!(
"`{path}` [{}]: stdout is not valid JSON\n stdout: {}\n stderr: {}",
args.join(" "),
String::from_utf8_lossy(&stdout).trim(),
String::from_utf8_lossy(&stderr).trim(),
));
}
}
}
}
let _ = std::fs::remove_dir_all(&dir);
eprintln!(
"fuzzer: checked {checked} leaves, skipped {} (denylist): {skipped:?}",
skipped.len()
);
assert!(
failures.is_empty(),
"{} leaf command(s) broke the --json contract:\n\n{}",
failures.len(),
failures.join("\n\n"),
);
}