@@ -1,0 +1,158 @@
#!/usr/bin/env bash
# Allocate and check Sunshine base ports for per-VM instances.
#
# Every Sunshine port is an offset from the base `port` (net::map_port): from port - 5 (GameStream
# HTTPS) to port + 21 (RTSP). Per-VM instances therefore use base ports 100 apart:
#   desktop Sunshine  47989 (slot 0, left free)
#   first VM          48089 (slot 1)
#   second VM         48189 (slot 2), and so on.
#
# Usage:
#   sunshine-qemu-port [--dir DIR] [--write] <vm>
#       Print the base port of <vm>: the `port` in DIR/<vm>.conf when it has one, otherwise the lowest
#       slot whose range doesn't overlap any other DIR/*.conf. The result only depends on the files in
#       DIR, so running it again gives the same answer. --write appends `port = <port>` to
#       DIR/<vm>.conf when the file has no port yet.
#   sunshine-qemu-port [--dir DIR] --check
#       Exit 1 and list the problems when two configs in DIR have overlapping port ranges or a range
#       leaves 1024-65535.
#
# Environment:
#   SUNSHINE_QEMU_CONF_DIR  default for --dir (default: /etc/sunshine-qemu)
#   SUNSHINE_QEMU_BASE_PORT port of slot 0 (default: 47989)
set -euo pipefail
readonly step=100
readonly low_offset=5   # port - 5 is the lowest port an instance binds
readonly high_offset=21 # port + 21 is the highest
dir="${SUNSHINE_QEMU_CONF_DIR:-/etc/sunshine-qemu}"
base="${SUNSHINE_QEMU_BASE_PORT:-47989}"
write=0
check=0
vm=""
usage() {
  sed -n '2,/^set -euo/p' "$0" | sed -e '/^set -euo/d' -e 's/^# \{0,1\}//' >&2
  exit 2
}
while (($#)); do
  case "$1" in
    --dir)
      dir="${2:?--dir needs a directory}"
      shift 2
      ;;
    --write)
      write=1
      shift
      ;;
    --check)
      check=1
      shift
      ;;
    -h | --help) usage ;;
    -*)
      echo "sunshine-qemu-port: unknown option $1" >&2
      usage
      ;;
    *)
      [[ -z "${vm}" ]] || usage
      vm="$1"
      shift
      ;;
  esac
done
# port_of <file>: the value of the last `port = N` line, or nothing
port_of() {
  sed -n -e 's/^[[:space:]]*port[[:space:]]*=[[:space:]]*\([0-9][0-9]*\)[[:space:]]*$/\1/p' "$1" | tail -n 1
}
# overlaps <a> <b>: whether the port ranges of two base ports share a port
overlaps() {
  local diff=$(($1 - $2))
  ((diff < 0)) && diff=$((-diff))
  ((diff <= low_offset + high_offset))
}
in_range() {
  (($1 - low_offset >= 1024 && $1 + high_offset <= 65535))
}
shopt -s nullglob
configs=("${dir}"/*.conf)
if ((check)); then
  [[ -z "${vm}" && ${write} == 0 ]] || usage
  status=0
  declare -A seen=()
  for conf in "${configs[@]}"; do
    port="$(port_of "${conf}")"
    if [[ -z "${port}" ]]; then
      echo "$(basename "${conf}"): no port set (Sunshine would use 47989)" >&2
      port=47989
    fi
    if ! in_range "${port}"; then
      echo "$(basename "${conf}"): port ${port} uses ports outside 1024-65535" >&2
      status=1
    fi
    for other in "${!seen[@]}"; do
      if overlaps "${port}" "${seen[${other}]}"; then
        echo "$(basename "${conf}") (port ${port}) overlaps ${other} (port ${seen[${other}]})" >&2
        status=1
      fi
    done
    seen["$(basename "${conf}")"]="${port}"
  done
  exit "${status}"
fi
[[ -n "${vm}" ]] || usage
if [[ ! "${vm}" =~ ^[A-Za-z0-9][A-Za-z0-9_.@-]*$ ]]; then
  echo "sunshine-qemu-port: invalid VM name '${vm}'" >&2
  exit 2
fi
own="${dir}/${vm}.conf"
if [[ -f "${own}" ]]; then
  port="$(port_of "${own}")"
  if [[ -n "${port}" ]]; then
    echo "${port}"
    exit 0
  fi
fi
used=()
for conf in "${configs[@]}"; do
  [[ "${conf}" == "${own}" ]] && continue
  port="$(port_of "${conf}")"
  used+=("${port:-47989}")
done
slot=1
while true; do
  candidate=$((base + slot * step))
  if ! in_range "${candidate}"; then
    echo "sunshine-qemu-port: no free port slot left in ${dir}" >&2
    exit 1
  fi
  free=1
  for port in "${used[@]}"; do
    if overlaps "${candidate}" "${port}"; then
      free=0
      break
    fi
  done
  ((free)) && break
  slot=$((slot + 1))
done
if ((write)); then
  if [[ ! -f "${own}" ]]; then
    echo "sunshine-qemu-port: ${own} doesn't exist" >&2
    exit 1
  fi
  printf 'port = %s\n' "${candidate}" >> "${own}"
fi
echo "${candidate}"

@@ -1,0 +1,125 @@
#!/usr/bin/env bash
# Start a private Avahi mDNS daemon for tests, without root and without touching the host's network.
#
# avahi-daemon runs in a new user, network and mount namespace (unshare -rnm) with only a dummy
# interface, so nothing is announced on a real network. It registers on a private dbus-daemon that
# stands in for the system bus: point Avahi clients (Sunshine, avahi-browse) at it with
# DBUS_SYSTEM_BUS_ADDRESS=<printed address>. The packages are downloaded once with apt-get download
# (no installation) and cached in E2E_CACHE.
#
# Usage: avahi_sandbox.sh <work-dir>
#   prints the bus address on success; exits 77 when the sandbox can't be created on this host
#
# Environment:
#   E2E_CACHE        cache for the extracted packages (default: ~/.cache/sunshine-qemu/e2e)
#   AVAHI_HOST_NAME  mDNS host name (default: sunshine-e2e-host)
#
# Writes into <work-dir>: bus.sock, bus.conf, dbus.pid, avahi.pid, avahi.log, dbus.log.
# Also prints the directory with avahi-browse on stderr as "avahi-bin: <dir>".
set -euo pipefail
work="${1:?usage: avahi_sandbox.sh <work-dir>}"
cache="${E2E_CACHE:-${HOME}/.cache/sunshine-qemu/e2e}/avahi"
host_name="${AVAHI_HOST_NAME:-sunshine-e2e-host}"
mkdir -p "${work}" "${cache}"
work="$(cd "${work}" && pwd)"
skip() {
  echo "avahi sandbox: SKIP: $*" >&2
  exit 77
}
root="${cache}/root"
if [[ ! -x "${root}/usr/sbin/avahi-daemon" || ! -x "${root}/usr/bin/avahi-browse" ]]; then
  command -v apt-get > /dev/null || skip "apt-get is needed to fetch avahi-daemon and avahi-utils"
  (
    cd "${cache}"
    apt-get download avahi-daemon avahi-utils libavahi-core7 libdaemon0 > /dev/null
    for deb in ./*.deb; do
      dpkg-deb -x "${deb}" root
    done
  ) || skip "couldn't download or extract the avahi packages"
fi
libs="${root}/usr/lib/x86_64-linux-gnu"
unshare -rnm true 2> /dev/null || skip "unprivileged user namespaces are not available (unshare -rnm)"
# avahi-daemon looks up its user even with --no-drop-root; inside the namespace it maps to us
{ cat /etc/passwd; echo "avahi:x:0:0:avahi:/nonexistent:/usr/sbin/nologin"; } > "${work}/passwd"
{ cat /etc/group; echo "avahi:x:0:"; } > "${work}/group"
# inside the user namespace we are uid 0, which EXTERNAL authentication can't prove to a bus outside
# it, so the private bus also accepts anonymous clients
cat > "${work}/bus.conf" <<EOF
<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-Bus Bus Configuration 1.0//EN"
 "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
  <type>system</type>
  <listen>unix:path=${work}/bus.sock</listen>
  <auth>EXTERNAL</auth>
  <auth>ANONYMOUS</auth>
  <allow_anonymous/>
  <policy context="default">
    <allow user="*"/>
    <allow send_destination="*" eavesdrop="true"/>
    <allow eavesdrop="true"/>
    <allow own="*"/>
  </policy>
</busconfig>
EOF
cat > "${work}/avahi-daemon.conf" <<EOF
[server]
host-name=${host_name}
use-ipv4=yes
use-ipv6=no
allow-interfaces=sq0
enable-dbus=yes
ratelimit-interval-usec=1000000
ratelimit-burst=1000
[wide-area]
enable-wide-area=no
[publish]
publish-hinfo=no
publish-workstation=no
EOF
rm -f "${work}/bus.sock"
dbus-daemon --config-file="${work}/bus.conf" --nofork --nopidfile > "${work}/dbus.log" 2>&1 &
echo $! > "${work}/dbus.pid"
for _ in $(seq 1 100); do
  [[ -S "${work}/bus.sock" ]] && break
  sleep 0.05
done
[[ -S "${work}/bus.sock" ]] || { echo "avahi sandbox: dbus-daemon didn't start" >&2; exit 1; }
address="unix:path=${work}/bus.sock"
# shellcheck disable=SC2016 # expanded by the inner shell
DBUS_SYSTEM_BUS_ADDRESS="${address}" LD_LIBRARY_PATH="${libs}" unshare -rnm sh -c '
  set -e
  ip link add sq0 type dummy
  ip link set sq0 multicast on up
  ip addr add 10.77.0.1/24 dev sq0
  mount -t tmpfs tmpfs /run
  mount --bind "$1/passwd" /etc/passwd
  mount --bind "$1/group" /etc/group
  exec "$2/usr/sbin/avahi-daemon" -f "$1/avahi-daemon.conf" --no-drop-root --no-chroot --no-proc-title --no-rlimits
' sh "${work}" "${root}" > "${work}/avahi.log" 2>&1 &
echo $! > "${work}/avahi.pid"
for _ in $(seq 1 100); do
  if ! kill -0 "$(cat "${work}/avahi.pid")" 2> /dev/null; then
    echo "avahi sandbox: avahi-daemon exited" >&2
    cat "${work}/avahi.log" >&2
    exit 1
  fi
  if grep -q "Server startup complete" "${work}/avahi.log"; then
    echo "avahi-bin: ${root}/usr/bin (LD_LIBRARY_PATH=${libs})" >&2
    echo "${address}"
    exit 0
  fi
  sleep 0.1
done
echo "avahi sandbox: avahi-daemon didn't finish starting" >&2
cat "${work}/avahi.log" >&2
exit 1

@@ -1,0 +1,340 @@
#!/usr/bin/env bash
# @tag requirements: [REQ-DEP-001]
# Multi-VM integration test (REQ-DEP-001): two VMs, two Sunshine instances and two Moonlight clients
# at the same time on one host, configured only through per-VM config files.
#
# 1. start a private Avahi daemon (avahi_sandbox.sh) so the instances' mDNS services can be browsed
# 2. start two pattern VMs, each with its own D-Bus display (VM 2 over QMP p2p with E2E_VM2_TRANSPORT=qmp)
# 3. write vm-1.conf and vm-2.conf into a config directory, allocate their base ports with
#    packaging/linux/qemu/sunshine-qemu-port and check the ranges don't overlap
# 4. start both Sunshine instances with the same HOME, so anything written to a hard-coded
#    ~/.config/sunshine path is shared and detected
# 5. pair one client with each instance and stream both at the same time; both must decode the pattern
# 6. assert:
#    - distinct mDNS service names, equal to each sunshine_name, advertising each instance's HTTP port
#    - every port an instance listens on lies in its own range and no port is used by both
#    - each client is paired with its own instance only (serverinfo PairStatus with the client certificate,
#      and the web API client list), and the two instances have different unique ids
#    - each instance streamed its own VM
#    - no PulseAudio null sinks were created (capture = qemu takes audio from QEMU)
#    - nothing but Sunshine's own directory was written under the shared ~/.config/sunshine
#
# Environment:
#   SUNSHINE_BIN, CLIENT_BIN, QEMU, E2E_CACHE, E2E_TIMEOUT, E2E_WIDTH/E2E_HEIGHT/E2E_FPS, E2E_ENCODER: as e2e_stream.sh
#   E2E_FRAMES         decoded frames per client (default: 90, so the two streams overlap)
#   E2E_MDNS           1 (default) requires the Avahi sandbox; 0 skips the mDNS assertions
#   E2E_VM2_TRANSPORT  bus (default: private dbus-daemon) or qmp (QEMU -display dbus,p2p=on; Sunshine connects
#                      with qemu_dbus_address = qmp:<socket>)
#   E2E_ARTIFACTS      where logs and summaries go (default: <work-dir>/artifacts, kept on failure)
#   E2E_KEEP           set to 1 to keep the work directory
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo="$(cd "${script_dir}/../../.." && pwd)"
sunshine_bin="${SUNSHINE_BIN:-${repo}/cmake-build-debug/sunshine}"
client_bin="${CLIENT_BIN:-${repo}/cmake-build-e2e-client/moonlight_e2e_client}"
port_helper="${repo}/packaging/linux/qemu/sunshine-qemu-port"
cache="${E2E_CACHE:-${HOME}/.cache/sunshine-qemu/e2e}"
timeout_s="${E2E_TIMEOUT:-120}"
width="${E2E_WIDTH:-1280}"
height="${E2E_HEIGHT:-800}"
fps="${E2E_FPS:-30}"
frames="${E2E_FRAMES:-90}"
encoder="${E2E_ENCODER:-software}"
mdns="${E2E_MDNS:-1}"
vm2_transport="${E2E_VM2_TRANSPORT:-bus}"
for bin in "${sunshine_bin}" "${client_bin}"; do
  if [[ ! -x "${bin}" ]]; then
    echo "missing ${bin}; build it first" >&2
    exit 2
  fi
done
case "${vm2_transport}" in
  bus | qmp) ;;
  *)
    echo "multi: E2E_VM2_TRANSPORT must be bus or qmp" >&2
    exit 2
    ;;
esac
work="$(mktemp -d "${TMPDIR:-/tmp}/sunshine-multi.XXXXXX")"
artifacts="${E2E_ARTIFACTS:-${work}/artifacts}"
mkdir -p "${artifacts}" "${work}/etc" "${work}/home"
result=1
failures=()
fail() {
  echo "multi: FAIL: $*" >&2
  failures+=("$*")
}
# shellcheck disable=SC2329 # called by the EXIT trap
cleanup() {
  set +e
  for pidfile in "${work}"/sunshine-*.pid "${work}"/client-*.pid "${work}"/vm-*/qemu.pid "${work}"/vm-*/dbus.pid "${work}/avahi/avahi.pid" "${work}/avahi/dbus.pid"; do
    [[ -f "${pidfile}" ]] || continue
    pid="$(cat "${pidfile}")"
    kill "${pid}" 2> /dev/null
    for _ in $(seq 1 50); do
      kill -0 "${pid}" 2> /dev/null || break
      sleep 0.1
    done
    kill -9 "${pid}" 2> /dev/null
  done
  for i in 1 2; do
    cp -f "${work}/sunshine-${i}.log" "${work}/sunshine-${i}.stdout" "${work}/client-${i}.log" "${artifacts}/" 2> /dev/null
    cp -f "${work}/vm-${i}/qemu.log" "${artifacts}/qemu-${i}.log" 2> /dev/null
    cp -f "${work}/etc/vm-${i}.conf" "${artifacts}/" 2> /dev/null
  done
  cp -f "${work}/avahi/avahi.log" "${artifacts}/" 2> /dev/null
  if [[ ${result} == 0 && "${E2E_KEEP:-0}" != 1 && -z "${E2E_ARTIFACTS:-}" ]]; then
    rm -rf "${work}"
  else
    echo "multi: work directory kept at ${work} (artifacts: ${artifacts})" >&2
  fi
}
trap cleanup EXIT
# 1. mDNS
avahi_bus=""
if [[ "${mdns}" == 1 ]]; then
  set +e
  avahi_bus="$(E2E_CACHE="${cache}" "${script_dir}/avahi_sandbox.sh" "${work}/avahi" 2> "${work}/avahi-sandbox.log")"
  status=$?
  set -e
  if [[ ${status} != 0 ]]; then
    cat "${work}/avahi-sandbox.log" >&2
    echo "multi: the Avahi sandbox is required for the mDNS checks (set E2E_MDNS=0 to skip them)" >&2
    exit "${status}"
  fi
  avahi_root="${cache}/avahi/root"
  echo "multi: Avahi sandbox on ${avahi_bus}" >&2
fi
# 2. VMs
image="${cache}/pattern-$(sha256sum "${script_dir}/guest/pattern.S" | cut -c1-16).img"
if [[ ! -f "${image}" ]]; then
  "${script_dir}/guest/build_guest.sh" "${image}" > /dev/null
fi
declare -A address
address[1]="$(VM_NAME=sq-vm-1 "${script_dir}/run_vm.sh" "${work}/vm-1" "${image}")"
address[2]="$(VM_NAME=sq-vm-2 VM_DBUS="${vm2_transport}" "${script_dir}/run_vm.sh" "${work}/vm-2" "${image}")"
echo "multi: VMs up on ${address[1]} and ${address[2]}" >&2
# 3. per-VM configuration, only through config files
declare -A port api_pass
for i in 1 2; do
  state="${work}/state-${i}"
  mkdir -p "${state}"
  cat > "${work}/etc/vm-${i}.conf" <<EOF
capture = qemu
qemu_dbus_address = ${address[${i}]}
encoder = ${encoder}
sunshine_name = sq-vm-${i}
file_state = ${state}/sunshine_state.json
credentials_file = ${state}/sunshine_credentials.json
pkey = ${state}/sunshine.key
cert = ${state}/sunshine.crt
file_apps = ${state}/apps.json
log_path = ${work}/sunshine-${i}.log
min_log_level = debug
upnp = disabled
system_tray = disabled
origin_web_ui_allowed = pc
notify_pre_releases = disabled
EOF
  cat > "${state}/apps.json" <<'EOF'
{"env": {}, "apps": [{"name": "Desktop", "image-path": "desktop.png"}]}
EOF
  port[${i}]="$("${port_helper}" --dir "${work}/etc" --write "vm-${i}")"
done
"${port_helper}" --dir "${work}/etc" --check
echo "multi: base ports ${port[1]} and ${port[2]}" >&2
# 4. Sunshine instances, sharing one HOME
run_sunshine() {
  exec env HOME="${work}/home" XDG_CONFIG_HOME= ${avahi_bus:+DBUS_SYSTEM_BUS_ADDRESS="${avahi_bus}"} "${sunshine_bin}" "$@"
}
for i in 1 2; do
  api_pass[${i}]="$(head -c 12 /dev/urandom | od -An -tx1 | tr -d ' \n')"
  (run_sunshine "${work}/etc/vm-${i}.conf" --creds e2e "${api_pass[${i}]}") > "${work}/creds-${i}.log" 2>&1
done
for i in 1 2; do
  # exec, so $! is Sunshine's pid (the port check matches sockets by pid)
  (run_sunshine "${work}/etc/vm-${i}.conf" > "${work}/sunshine-${i}.stdout" 2>&1) &
  echo $! > "${work}/sunshine-${i}.pid"
done
for i in 1 2; do
  for _ in $(seq 1 300); do
    if ! kill -0 "$(cat "${work}/sunshine-${i}.pid")" 2> /dev/null; then
      echo "multi: Sunshine ${i} exited early" >&2
      tail -50 "${work}/sunshine-${i}.stdout" >&2
      exit 1
    fi
    if curl -s --max-time 1 "http://127.0.0.1:${port[${i}]}/serverinfo" | grep -q 'status_code="200"'; then
      break
    fi
    sleep 0.1
  done
  echo "multi: Sunshine ${i} up on port ${port[${i}]}" >&2
done
# 5. pair and stream both at the same time
for i in 1 2; do
  "${client_bin}" \
    --port "${port[${i}]}" \
    --api-user e2e \
    --api-pass "${api_pass[${i}]}" \
    --state-dir "${work}/client-${i}" \
    --width "${width}" --height "${height}" --fps "${fps}" \
    --frames "${frames}" \
    --timeout "${timeout_s}" \
    --ready-file "${work}/client-${i}.ready" --ready-frames 15 \
    --out "${artifacts}/last_frame-${i}.ppm" \
    --summary "${artifacts}/summary-${i}.json" \
    --expect "255,0,0;0,255,0;0,0,255;255,255,255" --tolerance 48 \
    2> "${work}/client-${i}.log" &
  echo $! > "${work}/client-${i}.pid"
done
# while both streams run: every socket of each instance
both_streaming=0
for _ in $(seq 1 $((timeout_s * 10))); do
  if [[ -f "${work}/client-1.ready" && -f "${work}/client-2.ready" ]]; then
    both_streaming=1
    break
  fi
  if ! kill -0 "$(cat "${work}/client-1.pid")" 2> /dev/null && ! kill -0 "$(cat "${work}/client-2.pid")" 2> /dev/null; then
    break
  fi
  sleep 0.1
done
if ((both_streaming)); then
  ss -Htanup > "${work}/sockets.txt" 2> /dev/null || true
  cp "${work}/sockets.txt" "${artifacts}/"
  echo "multi: both clients are streaming" >&2
else
  fail "the two clients never streamed at the same time"
fi
client_status=()
for i in 1 2; do
  set +e
  wait "$(cat "${work}/client-${i}.pid")"
  client_status[i]=$?
  set -e
  rm -f "${work}/client-${i}.pid"
  if [[ ${client_status[${i}]} != 0 ]]; then
    fail "client ${i} exit ${client_status[${i}]}"
    tail -20 "${work}/client-${i}.log" >&2
  else
    echo "multi: stream ${i} passed: $(cat "${artifacts}/summary-${i}.json")" >&2
  fi
done
# 6a. ports: each instance only listens inside its own range, and never on a port of the other
if ((both_streaming)); then
  declare -A ports_of
  for i in 1 2; do
    pid="$(cat "${work}/sunshine-${i}.pid")"
    ports_of[${i}]="$(awk -v pid="pid=${pid}," '
      index($0, pid) && ($2 == "LISTEN" || $1 == "udp") {
        n = split($5, parts, ":"); print $1 "/" parts[n]
      }' "${work}/sockets.txt" | sort -u | tr '\n' ' ')"
    echo "multi: Sunshine ${i} sockets: ${ports_of[${i}]}" >&2
    [[ -n "${ports_of[${i}]}" ]] || fail "no sockets found for Sunshine ${i}"
    for entry in ${ports_of[${i}]}; do
      p="${entry#*/}"
      if ((p < port[${i}] - 5 || p > port[${i}] + 21)); then
        fail "Sunshine ${i} (port ${port[${i}]}) listens on ${entry}, outside its range"
      fi
    done
    for expected in tcp/$((port[${i}] - 5)) tcp/${port[${i}]} tcp/$((port[${i}] + 1)) tcp/$((port[${i}] + 21)) udp/$((port[${i}] + 9)) udp/$((port[${i}] + 10)) udp/$((port[${i}] + 11)); do
      [[ " ${ports_of[${i}]} " == *" ${expected} "* ]] || fail "Sunshine ${i} doesn't listen on ${expected}"
    done
  done
  for entry in ${ports_of[1]}; do
    [[ " ${ports_of[2]} " != *" ${entry} "* ]] || fail "both instances use ${entry}"
  done
fi
for i in 1 2; do
  if grep -qiE "couldn't bind|address already in use" "${work}/sunshine-${i}.log"; then
    fail "Sunshine ${i} logged a bind failure: $(grep -iE "couldn't bind|address already in use" "${work}/sunshine-${i}.log" | head -1)"
  fi
done
# 6b. pairing is per instance: with its certificate a client gets serverinfo (PairStatus 1) from its own
# instance, while the other instance refuses the certificate (HTTP 401) or reports it unpaired
declare -A unique_id
for j in 1 2; do
  unique_id[${j}]="$(curl -s --max-time 5 "http://127.0.0.1:${port[${j}]}/serverinfo" | sed -n 's:.*<uniqueid>\([^<]*\)</uniqueid>.*:\1:p')"
done
for i in 1 2; do
  for j in 1 2; do
    info="$(curl -sk --max-time 5 --cert "${work}/client-${i}/client.pem" --key "${work}/client-${i}/client.key" \
      "https://127.0.0.1:$((port[${j}] - 5))/serverinfo?uniqueid=$(cat "${work}/client-${i}/uniqueid")" || true)"
    status="$(sed -n 's:.*<root status_code="\([0-9]*\)".*:\1:p' <<< "${info}")"
    paired="$(sed -n 's:.*<PairStatus>\([01]\)</PairStatus>.*:\1:p' <<< "${info}")"
    if [[ ${i} == "${j}" ]]; then
      [[ "${status}" == 200 && "${paired}" == 1 ]] || fail "client ${i} isn't paired with its own instance (status '${status}', PairStatus '${paired}')"
    elif [[ "${status}" == 200 && "${paired}" != 0 ]] || [[ "${status}" != 200 && "${status}" != 401 ]]; then
      fail "client ${i} on instance ${j}: status '${status}', PairStatus '${paired}', expected unpaired"
    fi
  done
  clients="$(curl -sk --max-time 5 -u "e2e:${api_pass[${i}]}" "https://127.0.0.1:$((port[${i}] + 1))/api/clients/list" || true)"
  count="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("named_certs", [])))' <<< "${clients}" 2> /dev/null || echo "?")"
  [[ "${count}" == 1 ]] || fail "instance ${i} has ${count} paired clients, expected 1: ${clients}"
done
if [[ -z "${unique_id[1]}" || "${unique_id[1]}" == "${unique_id[2]}" ]]; then
  fail "the instances don't have distinct unique ids ('${unique_id[1]}', '${unique_id[2]}')"
fi
# 6c. each instance streamed its own VM
for i in 1 2; do
  grep -q "qemu: connected to VM \[sq-vm-${i}\]" "${work}/sunshine-${i}.log" || fail "Sunshine ${i} never connected to sq-vm-${i}"
  other=$((3 - i))
  if grep -q "qemu: connected to VM \[sq-vm-${other}\]" "${work}/sunshine-${i}.log"; then
    fail "Sunshine ${i} connected to sq-vm-${other}"
  fi
done
# 6d. mDNS: one service per instance, named after sunshine_name, on that instance's HTTP port
if [[ -n "${avahi_bus}" ]]; then
  services="$(DBUS_SYSTEM_BUS_ADDRESS="${avahi_bus}" LD_LIBRARY_PATH="${avahi_root}/usr/lib/x86_64-linux-gnu" \
    timeout 10 "${avahi_root}/usr/bin/avahi-browse" -tprk _nvstream._tcp 2> "${work}/avahi-browse.log" | grep '^=' | sort -u || true)"
  echo "${services}" > "${artifacts}/mdns.txt"
  names="$(cut -d';' -f4 <<< "${services}" | sort -u | tr '\n' ' ')"
  echo "multi: mDNS services: ${names}" >&2
  for i in 1 2; do
    grep -qE "^=;[^;]*;IPv4;sq-vm-${i};_nvstream\._tcp;local;[^;]*;[^;]*;${port[${i}]};" <<< "${services}" \
      || fail "no mDNS service 'sq-vm-${i}' on port ${port[${i}]} (found: ${names:-none})"
  done
  count="$(cut -d';' -f4 <<< "${services}" | sort -u | grep -c . || true)"
  [[ "${count}" == 2 ]] || fail "expected 2 mDNS services, found ${count}: ${names}"
  if grep -h "Avahi service name collision\|Service name collision" "${work}"/sunshine-*.log; then
    fail "an instance renamed its mDNS service after a collision"
  fi
fi
# 6e. no PulseAudio null sinks with capture = qemu
for i in 1 2; do
  # audio.cpp logs these at debug level when it connects to PulseAudio and loads its null sinks
  if grep -qE "null-sink args|Connecting to pulseaudio|virtual sink" "${work}/sunshine-${i}.log"; then
    fail "Sunshine ${i} used PulseAudio: $(grep -E "null-sink args|Connecting to pulseaudio|virtual sink" "${work}/sunshine-${i}.log" | head -1)"
  fi
done
# 6f. nothing shared under the hard-coded appdata directory
find "${work}/home" -mindepth 1 | sed "s:^${work}/home:~:" > "${artifacts}/home-files.txt"
unexpected="$(grep -v -E '^~/\.config$|^~/\.config/sunshine$' "${artifacts}/home-files.txt" || true)"
[[ -z "${unexpected}" ]] || fail "files written under the shared HOME: $(tr '\n' ' ' <<< "${unexpected}")"
if ((${#failures[@]} == 0)); then
  result=0
  echo "multi: PASS {\"ports\":[${port[1]},${port[2]}],\"vm2_transport\":\"${vm2_transport}\",\"mdns\":$([[ -n "${avahi_bus}" ]] && echo true || echo false)}"
else
  echo "multi: FAIL (${#failures[@]} problem(s))" >&2
fi
exit "${result}"