@@ -14,6 +14,10 @@
* To test events during a stream (a guest reboot, for example), `--ready-file` is created once
* `--ready-frames` frames are decoded, and with `--wait-file` the `--frames` count and the picture
* check only start once that file exists.
*
* With `--audio-freq`, the client also decodes the Opus audio stream and requires
* `--audio-seconds` of audio whose dominant frequency, in each channel, is within
* `--audio-tolerance` Hz of the expected one.
*/
// standard includes
#include <algorithm>
@@ -21,12 +25,14 @@
#include <atomic>
#include <chrono>
#include <cmath>
#include <complex>
#include <cstdarg>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <mutex>
#include <numbers>
#include <optional>
#include <regex>
#include <sstream>
@@ -40,6 +46,7 @@
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <Limelight.h>
#include <opus/opus_multistream.h>
}
// local includes
@@ -71,6 +78,9 @@
std::string ready_file; ///< File created once `ready_frames` frames were decoded.
int ready_frames {1}; ///< Decoded frames before `ready_file` is created.
std::string wait_file; ///< Don't start counting `min_frames` until this file exists.
int audio_freq {0}; ///< Expected dominant audio frequency in hertz; 0 skips the audio check.
int audio_tolerance {10}; ///< Allowed difference from `audio_freq` in hertz.
int audio_seconds_x10 {20}; ///< Decoded audio required for the check, in tenths of a second.
};
/**
@@ -97,6 +107,187 @@
stream_state_t state;
/**
* @brief Decoded audio shared with the audio renderer callbacks.
*/
struct audio_state_t {
std::mutex mutex; ///< Guards the fields below.
OpusMSDecoder *decoder {nullptr}; ///< Opus multistream decoder.
int channels {0}; ///< Decoded channels.
int sample_rate {0}; ///< Decoded sample rate.
int samples_per_frame {0}; ///< Samples per Opus packet.
std::vector<float> pcm; ///< Decoded interleaved samples.
int packets {0}; ///< Decoded packets.
int lost {0}; ///< Packets the library reported as lost (concealed).
int errors {0}; ///< Packets that failed to decode.
};
audio_state_t audio;
int ar_init(int audio_configuration, const POPUS_MULTISTREAM_CONFIGURATION opus, void *context, int flags) {
std::lock_guard lock {audio.mutex};
int error = 0;
audio.decoder = opus_multistream_decoder_create(opus->sampleRate, opus->channelCount, opus->streams, opus->coupledStreams, opus->mapping, &error);
if (!audio.decoder) {
std::fprintf(stderr, "e2e: couldn't create the Opus decoder: %s\n", opus_strerror(error));
return -1;
}
audio.channels = opus->channelCount;
audio.sample_rate = opus->sampleRate;
audio.samples_per_frame = opus->samplesPerFrame;
std::fprintf(stderr, "e2e: audio decoder set up for %d channel(s) at %d Hz, %d samples per packet\n", opus->channelCount, opus->sampleRate, opus->samplesPerFrame);
return 0;
}
void ar_cleanup() {
std::lock_guard lock {audio.mutex};
if (audio.decoder) {
opus_multistream_decoder_destroy(audio.decoder);
audio.decoder = nullptr;
}
}
void ar_decode_and_play(char *data, int length) {
std::lock_guard lock {audio.mutex};
if (!audio.decoder) {
return;
}
std::vector<float> frame((std::size_t) audio.samples_per_frame * audio.channels);
const int decoded = opus_multistream_decode_float(audio.decoder, (const unsigned char *) data, data ? length : 0, frame.data(), audio.samples_per_frame, 0);
if (decoded < 0) {
audio.errors += 1;
return;
}
if (!data) {
audio.lost += 1;
} else {
audio.packets += 1;
}
audio.pcm.insert(audio.pcm.end(), frame.begin(), frame.begin() + (std::ptrdiff_t) decoded * audio.channels);
}
/**
* @brief Result of analyzing the decoded audio.
*/
struct audio_result_t {
double seconds {0}; ///< Decoded audio duration.
std::vector<double> dominant_hz; ///< Dominant frequency of each channel over the analyzed window.
double rms {0}; ///< RMS of all channels over the window.
int silent_blocks {0}; ///< 10 ms blocks in the window whose RMS is below -40 dBFS.
int blocks {0}; ///< 10 ms blocks in the window.
};
/**
* @brief Dominant frequency of one channel, from a Hann-windowed FFT with peak interpolation.
*
* @param samples Samples of the channel.
* @param sample_rate Sample rate in hertz.
* @return Frequency in hertz, or 0 for silence.
*/
double dominant_frequency(const std::vector<float> &samples, int sample_rate) {
if (samples.size() < 2) {
return 0;
}
std::size_t n = 1;
while (n < samples.size()) {
n <<= 1;
}
std::vector<std::complex<double>> a(n);
for (std::size_t i = 0; i < samples.size(); ++i) {
a[i] = samples[i] * (0.5 - 0.5 * std::cos(2 * std::numbers::pi * i / (samples.size() - 1)));
}
for (std::size_t i = 1, j = 0; i < n; ++i) {
std::size_t bit = n >> 1;
for (; j & bit; bit >>= 1) {
j ^= bit;
}
j ^= bit;
if (i < j) {
std::swap(a[i], a[j]);
}
}
for (std::size_t len = 2; len <= n; len <<= 1) {
const std::complex<double> wlen {std::cos(-2 * std::numbers::pi / len), std::sin(-2 * std::numbers::pi / len)};
for (std::size_t i = 0; i < n; i += len) {
std::complex<double> w {1};
for (std::size_t j = 0; j < len / 2; ++j) {
auto u = a[i + j];
auto v = a[i + j + len / 2] * w;
a[i + j] = u + v;
a[i + j + len / 2] = u - v;
w *= wlen;
}
}
}
std::size_t peak = 1;
for (std::size_t i = 1; i < n / 2; ++i) {
if (std::abs(a[i]) > std::abs(a[peak])) {
peak = i;
}
}
if (std::abs(a[peak]) < 1e-6) {
return 0;
}
const double l = std::abs(a[peak - 1]);
const double c = std::abs(a[peak]);
const double r = std::abs(a[peak + 1]);
const double d = l - 2 * c + r;
return (peak + (d == 0 ? 0 : 0.5 * (l - r) / d)) * sample_rate / n;
}
/**
* @brief Analyze the most recent audio.
*
* @param seconds Length of the window at the end of the decoded audio.
* @return Analysis of the window.
*/
audio_result_t analyze_audio(double seconds) {
std::lock_guard lock {audio.mutex};
audio_result_t result;
if (audio.channels == 0 || audio.sample_rate == 0) {
return result;
}
const std::size_t frames = audio.pcm.size() / audio.channels;
result.seconds = (double) frames / audio.sample_rate;
const std::size_t window = std::min(frames, (std::size_t) (seconds * audio.sample_rate));
const std::size_t first = frames - window;
double sum = 0;
for (int c = 0; c < audio.channels; ++c) {
std::vector<float> channel(window);
for (std::size_t i = 0; i < window; ++i) {
channel[i] = audio.pcm[(first + i) * audio.channels + c];
sum += (double) channel[i] * channel[i];
}
result.dominant_hz.push_back(dominant_frequency(channel, audio.sample_rate));
}
result.rms = window ? std::sqrt(sum / (window * audio.channels)) : 0;
const std::size_t block = audio.sample_rate / 100;
for (std::size_t b = 0; b + block <= window; b += block) {
double block_sum = 0;
for (std::size_t i = (first + b) * audio.channels; i < (first + b + block) * audio.channels; ++i) {
block_sum += (double) audio.pcm[i] * audio.pcm[i];
}
result.blocks += 1;
if (std::sqrt(block_sum / (block * audio.channels)) < 0.01) {
result.silent_blocks += 1;
}
}
return result;
}
/**
* @brief Seconds of audio decoded so far.
*
* @return Decoded duration.
*/
double decoded_audio_seconds() {
std::lock_guard lock {audio.mutex};
if (audio.channels == 0 || audio.sample_rate == 0) {
return 0;
}
return (double) audio.pcm.size() / audio.channels / audio.sample_rate;
}
int dr_setup(int video_format, int width, int height, int redraw_rate, void *context, int dr_flags) {
auto codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec) {
@@ -368,6 +559,9 @@
{"--timeout", &opts.timeout_s},
{"--tolerance", &opts.tolerance},
{"--ready-frames", &opts.ready_frames},
{"--audio-freq", &opts.audio_freq},
{"--audio-tolerance", &opts.audio_tolerance},
{"--audio-seconds-x10", &opts.audio_seconds_x10},
};
for (int i = 1; i < argc; ++i) {
std::string arg = argv[i];
@@ -489,9 +683,16 @@
decoder.setup = dr_setup;
decoder.cleanup = dr_cleanup;
decoder.submitDecodeUnit = dr_submit;
AUDIO_RENDERER_CALLBACKS audio_renderer;
LiInitializeAudioCallbacks(&audio_renderer);
audio_renderer.init = ar_init;
audio_renderer.cleanup = ar_cleanup;
audio_renderer.decodeAndPlaySample = ar_decode_and_play;
const double audio_seconds = opts.audio_seconds_x10 / 10.0;
const auto stream_start = std::chrono::steady_clock::now();
if (LiStartConnection(&server, &config, &listener, &decoder, nullptr, nullptr, 0, nullptr, 0) != 0) {
if (LiStartConnection(&server, &config, &listener, &decoder, &audio_renderer, nullptr, 0, nullptr, 0) != 0) {
std::fprintf(stderr, "e2e: LiStartConnection failed\n");
client.cancel();
return 1;
@@ -524,6 +725,9 @@
if (state.decoded - frames_at_wait.value_or(0) < opts.min_frames || state.rgb.empty()) {
continue;
}
if (opts.audio_freq > 0 && decoded_audio_seconds() < audio_seconds) {
continue;
}
if (!expected) {
matched = true;
break;
@@ -540,6 +744,24 @@
LiStopConnection();
client.cancel();
// audio: the last `audio_seconds` must carry the expected tone on every channel
const auto audio_result = analyze_audio(audio_seconds);
bool audio_matched = true;
if (opts.audio_freq > 0) {
audio_matched = audio_result.seconds >= audio_seconds && !audio_result.dominant_hz.empty();
for (auto hz : audio_result.dominant_hz) {
audio_matched = audio_matched && std::abs(hz - opts.audio_freq) <= opts.audio_tolerance;
}
std::string frequencies;
for (auto hz : audio_result.dominant_hz) {
frequencies += (frequencies.empty() ? "" : "/") + std::to_string(hz);
}
std::fprintf(stderr, "e2e: audio %.2f s decoded, dominant %s Hz, rms %.4f, %d of %d 10 ms blocks silent\n", audio_result.seconds, frequencies.c_str(), audio_result.rms, audio_result.silent_blocks, audio_result.blocks);
if (!audio_matched) {
std::fprintf(stderr, "e2e: audio doesn't match: expected %d Hz +- %d Hz over %.1f s\n", opts.audio_freq, opts.audio_tolerance, audio_seconds);
}
}
std::vector<double> latencies;
int decoded = 0;
int picture_changes = 0;
@@ -566,11 +788,19 @@
for (std::size_t i = 0; i < last_colors.size(); ++i) {
summary << (i ? "," : "") << "[" << last_colors[i][0] << "," << last_colors[i][1] << "," << last_colors[i][2] << "]";
}
summary << "],\"frames_after_wait\":" << (decoded - frames_at_wait.value_or(0)) << ",\"picture_changes_after_wait\":" << picture_changes << ",\"max_frame_gap_ms\":" << max_gap_ms << ",\"terminated\":" << (state.terminated ? "true" : "false");
{
summary << "],\"frames_after_wait\":" << (decoded - frames_at_wait.value_or(0)) << ",\"picture_changes_after_wait\":" << picture_changes << ",\"max_frame_gap_ms\":" << max_gap_ms << ",\"terminated\":" << (state.terminated ? "true" : "false") << "}";
std::lock_guard lock {audio.mutex};
summary << ",\"audio\":{\"matched\":" << (opts.audio_freq > 0 ? (audio_matched ? "true" : "false") : "null") << ",\"packets\":" << audio.packets << ",\"lost_packets\":" << audio.lost << ",\"decode_errors\":" << audio.errors << ",\"channels\":" << audio.channels << ",\"seconds\":" << audio_result.seconds << ",\"dominant_hz\":[";
}
for (std::size_t i = 0; i < audio_result.dominant_hz.size(); ++i) {
summary << (i ? "," : "") << audio_result.dominant_hz[i];
}
summary << "],\"rms\":" << audio_result.rms << ",\"silent_blocks\":" << audio_result.silent_blocks << ",\"blocks\":" << audio_result.blocks << "}}";
std::printf("%s\n", summary.str().c_str());
if (!opts.summary.empty()) {
std::ofstream {opts.summary} << summary.str() << "\n";
}
return matched && audio_matched ? 0 : 1;
return matched ? 0 : 1;
}