@@ -3,6 +3,7 @@
* @brief Definitions for video.
*/
// standard includes
#include <algorithm>
#include <array>
#include <atomic>
#include <bitset>
@@ -194,198 +195,189 @@
*/
util::Either<avcodec_buffer_t, int> vulkan_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *);
/**
* @brief FFmpeg software encode device used when no hardware frames are required.
*/
class avcodec_software_encode_device_t: public platf::avcodec_encode_device_t {
public:
/**
* @brief Accept a software frame without additional hardware conversion.
*
* @param img Image or frame object to read from or populate.
* @return Conversion status.
*/
int convert(platf::img_t &img) override {
// If we need to add aspect ratio padding, we need to scale into an intermediate output buffer
bool requires_padding = (sw_frame->width != sws_output_frame->width || sw_frame->height != sws_output_frame->height);
int avcodec_software_encode_device_t::convert(platf::img_t &img) {
// If we need to add aspect ratio padding, we need to scale into an intermediate output buffer
bool requires_padding = (sw_frame->width != sws_output_frame->width || sw_frame->height != sws_output_frame->height);
// Detect the actual capture pixel format. PipeWire-based captures (KWin
// screencast / XDG portal) deliver NV12 with 1 byte per pixel, while
// Setup the input frame using the caller's img_t
sws_input_frame->data[0] = img.data;
sws_input_frame->linesize[0] = img.row_pitch;
// KMS/DMABUF captures deliver BGR0 (4 bytes per pixel). The capture
// backend reports bytes per pixel in img.pixel_pitch; fall back to the row
// pitch heuristic when it is unavailable.
const auto pixel_pitch = img.pixel_pitch > 0 ? img.pixel_pitch : (img.row_pitch / std::max(img.width, 1));
const auto input_fmt = (pixel_pitch == 1) ? AV_PIX_FMT_NV12 : AV_PIX_FMT_BGR0;
// The sws context is created with the default BGR0 source format;
// recreate it once if the capture is actually NV12.
// Perform color conversion and scaling to the final size
auto status = sws_scale_frame(sws.get(), requires_padding ? sws_output_frame.get() : sw_frame.get(), sws_input_frame.get());
if (input_fmt != sws_src_format) {
sws_src_format = input_fmt;
if (reinit_sws(input_fmt) < 0) {
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Couldn't scale frame: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
// The colorspace details were applied to the previous sws context.
apply_colorspace();
}
// Setup the input frame using the caller's img_t
sws_input_frame->data[0] = img.data;
sws_input_frame->linesize[0] = img.row_pitch;
if (input_fmt == AV_PIX_FMT_NV12) {
sws_input_frame->data[1] = img.data + static_cast<std::size_t>(img.row_pitch) * img.height;
sws_input_frame->linesize[1] = img.row_pitch;
} else {
sws_input_frame->data[1] = nullptr;
// If we require aspect ratio padding, copy the output frame into the final padded frame
if (requires_padding) {
auto fmt_desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(sws_output_frame->format));
auto planes = av_pix_fmt_count_planes(static_cast<AVPixelFormat>(sws_output_frame->format));
for (int plane = 0; plane < planes; plane++) {
auto shift_h = plane == 0 ? 0 : fmt_desc->log2_chroma_h;
auto shift_w = plane == 0 ? 0 : fmt_desc->log2_chroma_w;
auto offset = ((offsetW >> shift_w) * fmt_desc->comp[plane].step) + (offsetH >> shift_h) * sw_frame->linesize[plane];
sws_input_frame->linesize[1] = 0;
}
sws_input_frame->data[2] = nullptr;
sws_input_frame->linesize[2] = 0;
sws_input_frame->data[3] = nullptr;
sws_input_frame->linesize[3] = 0;
// Perform color conversion and scaling to the final size
auto status = sws_scale_frame(sws.get(), requires_padding ? sws_output_frame.get() : sw_frame.get(), sws_input_frame.get());
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Couldn't scale frame: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
// Copy line-by-line to preserve leading padding for each row
for (int line = 0; line < sws_output_frame->height >> shift_h; line++) {
memcpy(sw_frame->data[plane] + offset + (line * sw_frame->linesize[plane]), sws_output_frame->data[plane] + (line * sws_output_frame->linesize[plane]), static_cast<std::size_t>(sws_output_frame->width >> shift_w) * fmt_desc->comp[plane].step);
return -1;
}
}
}
}
// If frame is not a software frame, it means we still need to transfer from main memory
// to vram memory
if (frame->hw_frames_ctx) {
auto status = av_hwframe_transfer_data(frame, sw_frame.get(), 0);
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to transfer image data to hardware frame: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
// If we require aspect ratio padding, copy the output frame into the final padded frame
if (requires_padding) {
auto fmt_desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(sws_output_frame->format));
auto planes = av_pix_fmt_count_planes(static_cast<AVPixelFormat>(sws_output_frame->format));
for (int plane = 0; plane < planes; plane++) {
auto shift_h = plane == 0 ? 0 : fmt_desc->log2_chroma_h;
auto shift_w = plane == 0 ? 0 : fmt_desc->log2_chroma_w;
auto offset = ((offsetW >> shift_w) * fmt_desc->comp[plane].step) + (offsetH >> shift_h) * sw_frame->linesize[plane];
// Copy line-by-line to preserve leading padding for each row
for (int line = 0; line < sws_output_frame->height >> shift_h; line++) {
memcpy(sw_frame->data[plane] + offset + (line * sw_frame->linesize[plane]), sws_output_frame->data[plane] + (line * sws_output_frame->linesize[plane]), static_cast<std::size_t>(sws_output_frame->width >> shift_w) * fmt_desc->comp[plane].step);
}
}
}
return 0;
// If frame is not a software frame, it means we still need to transfer from main memory
// to vram memory
if (frame->hw_frames_ctx) {
auto status = av_hwframe_transfer_data(frame, sw_frame.get(), 0);
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to transfer image data to hardware frame: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
}
/**
* @brief Attach frame resources used by the next conversion or encode operation.
*
* @param frame Video or graphics frame being processed.
return 0;
}
int avcodec_software_encode_device_t::set_frame(AVFrame *in_frame, AVBufferRef *hw_frames_ctx) {
this->frame = in_frame;
* @param hw_frames_ctx FFmpeg hardware frames context associated with the frame.
* @return Status from updating frame.
*/
int set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override {
this->frame = frame;
// If it's a hwframe, allocate buffers for hardware
if (hw_frames_ctx) {
// If it's a hwframe, allocate buffers for hardware
if (hw_frames_ctx) {
hw_frame.reset(in_frame);
hw_frame.reset(frame);
if (av_hwframe_get_buffer(hw_frames_ctx, in_frame, 0)) {
return -1;
if (av_hwframe_get_buffer(hw_frames_ctx, frame, 0)) {
return -1;
}
} else {
sw_frame.reset(frame);
}
return 0;
} else {
sw_frame.reset(in_frame);
}
/**
* @brief Apply the configured colorspace metadata to the active frame.
*/
void apply_colorspace() override {
auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace);
return 0;
}
sws_setColorspaceDetails(sws.get(), sws_getCoefficients(SWS_CS_DEFAULT), 0, sws_getCoefficients(avcodec_colorspace.software_format), avcodec_colorspace.range - 1, 0, 1 << 16, 1 << 16);
}
/**
* When preserving aspect ratio, ensure that padding is black
*/
void prefill() {
auto frame = sw_frame ? sw_frame.get() : this->frame;
av_frame_get_buffer(frame, 0);
av_frame_make_writable(frame);
ptrdiff_t linesize[4] = {frame->linesize[0], frame->linesize[1], frame->linesize[2], frame->linesize[3]};
av_image_fill_black(frame->data, linesize, static_cast<AVPixelFormat>(frame->format), frame->color_range, frame->width, frame->height);
}
void avcodec_software_encode_device_t::apply_colorspace() {
auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace);
sws_setColorspaceDetails(sws.get(), sws_getCoefficients(SWS_CS_DEFAULT), 0, sws_getCoefficients(avcodec_colorspace.software_format), avcodec_colorspace.range - 1, 0, 1 << 16, 1 << 16);
}
/**
* @brief Initialize FFmpeg software encoding for the requested codec.
*
* @param in_width In width.
* @param in_height In height.
* @param frame Video or graphics frame being processed.
void avcodec_software_encode_device_t::prefill() {
auto active_frame = sw_frame ? sw_frame.get() : this->frame;
av_frame_get_buffer(active_frame, 0);
av_frame_make_writable(active_frame);
std::array<ptrdiff_t, 4> linesize = {active_frame->linesize[0], active_frame->linesize[1], active_frame->linesize[2], active_frame->linesize[3]};
av_image_fill_black(active_frame->data, linesize.data(), static_cast<AVPixelFormat>(active_frame->format), active_frame->color_range, active_frame->width, active_frame->height);
}
* @param format Pixel, audio, or protocol format being converted.
* @param hardware Whether the frame is backed by hardware resources.
* @return 0 on success; nonzero or negative platform status on failure.
*/
int init(int in_width, int in_height, AVFrame *frame, AVPixelFormat format, bool hardware) {
// If the device used is hardware, yet the image resides on main memory
if (hardware) {
sw_frame.reset(av_frame_alloc());
int avcodec_software_encode_device_t::init(int in_width, int in_height, AVFrame *in_frame, AVPixelFormat format, bool hardware) {
// If the device used is hardware, yet the image resides on main memory
sw_frame->width = frame->width;
sw_frame->height = frame->height;
sw_frame->format = format;
if (hardware) {
sw_frame.reset(av_frame_alloc());
} else {
this->frame = frame;
}
sw_frame->width = in_frame->width;
// Fill aspect ratio padding in the destination frame
prefill();
sw_frame->height = in_frame->height;
sw_frame->format = format;
} else {
this->frame = in_frame;
}
// Fill aspect ratio padding in the destination frame
auto out_width = frame->width;
prefill();
auto out_height = frame->height;
auto out_width = in_frame->width;
auto out_height = in_frame->height;
// Ensure aspect ratio is maintained
auto scalar = std::fminf(static_cast<float>(out_width) / in_width, static_cast<float>(out_height) / in_height);
out_width = in_width * scalar;
out_height = in_height * scalar;
sws_input_frame.reset(av_frame_alloc());
sws_input_frame->width = in_width;
sws_input_frame->height = in_height;
sws_input_frame->format = AV_PIX_FMT_BGR0;
// Ensure aspect ratio is maintained
auto scalar = std::fminf(static_cast<float>(out_width) / in_width, static_cast<float>(out_height) / in_height);
out_width = in_width * scalar;
out_height = in_height * scalar;
sws_input_frame.reset(av_frame_alloc());
sws_input_frame->width = in_width;
sws_input_frame->height = in_height;
sws_input_frame->format = AV_PIX_FMT_BGR0;
sws_output_frame.reset(av_frame_alloc());
sws_output_frame->width = out_width;
sws_output_frame->height = out_height;
sws_output_frame->format = format;
sws_output_frame.reset(av_frame_alloc());
// Result is always positive
offsetW = (frame->width - out_width) / 2;
offsetH = (frame->height - out_height) / 2;
sws_output_frame->width = out_width;
sws_output_frame->height = out_height;
sws_output_frame->format = format;
sws.reset(sws_alloc_context());
if (!sws) {
return -1;
// Result is always positive
offsetW = (in_frame->width - out_width) / 2;
offsetH = (in_frame->height - out_height) / 2;
}
AVDictionary *options {nullptr};
av_dict_set_int(&options, "srcw", sws_input_frame->width, 0);
av_dict_set_int(&options, "srch", sws_input_frame->height, 0);
av_dict_set_int(&options, "src_format", sws_input_frame->format, 0);
av_dict_set_int(&options, "dstw", sws_output_frame->width, 0);
av_dict_set_int(&options, "dsth", sws_output_frame->height, 0);
av_dict_set_int(&options, "dst_format", sws_output_frame->format, 0);
av_dict_set_int(&options, "sws_flags", SWS_LANCZOS | SWS_ACCURATE_RND, 0);
sws_src_format = AV_PIX_FMT_BGR0;
av_dict_set_int(&options, "threads", config::video.min_threads, 0);
return reinit_sws(sws_src_format);
auto status = av_opt_set_dict(sws.get(), &options);
av_dict_free(&options);
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to set SWS options: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
}
int avcodec_software_encode_device_t::reinit_sws(AVPixelFormat src_format) {
sws_input_frame->format = src_format;
status = sws_init_context(sws.get(), nullptr, nullptr);
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to initialize SWS: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
sws.reset(sws_alloc_context());
if (!sws) {
return -1;
return 0;
}
AVDictionary *options {nullptr};
av_dict_set_int(&options, "srcw", sws_input_frame->width, 0);
av_dict_set_int(&options, "srch", sws_input_frame->height, 0);
av_dict_set_int(&options, "src_format", src_format, 0);
av_dict_set_int(&options, "dstw", sws_output_frame->width, 0);
av_dict_set_int(&options, "dsth", sws_output_frame->height, 0);
av_dict_set_int(&options, "dst_format", sws_output_frame->format, 0);
av_dict_set_int(&options, "sws_flags", SWS_LANCZOS | SWS_ACCURATE_RND, 0);
av_dict_set_int(&options, "threads", config::video.min_threads, 0);
// Store ownership when frame is hw_frame
avcodec_frame_t hw_frame; ///< Hw frame.
auto status = av_opt_set_dict(sws.get(), &options);
av_dict_free(&options);
if (status < 0) {
avcodec_frame_t sw_frame; ///< Sw frame.
avcodec_frame_t sws_input_frame; ///< Sws input frame.
avcodec_frame_t sws_output_frame; ///< Sws output frame.
sws_t sws; ///< Software scaler used when frames need CPU-side pixel conversion.
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to set SWS options: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
status = sws_init_context(sws.get(), nullptr, nullptr);
if (status < 0) {
char string[AV_ERROR_MAX_STRING_SIZE];
BOOST_LOG(error) << "Failed to initialize SWS: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status);
return -1;
}
// Offset of input image to output frame in pixels
int offsetW; ///< Offset w.
return 0;
}
int offsetH; ///< Offset h.
};
/**
* @brief Enumerates supported flag options.
@@ -1399,7 +1391,7 @@
};
#endif
static const std::vector<encoder_t *> encoders {
static const std::vector encoders {
#ifndef __APPLE__
&nvenc,
#endif