From d3ecc741c6a15b8f8da82ef92d2725d034bf1eee Mon Sep 17 00:00:00 2001 From: silas <s.developer@4-dc.de> Date: Sun, 4 Jul 2021 16:44:41 +0200 Subject: [PATCH] Added simple configuration options - Muxer: output format - Encoders: codec and bitrate Further changes: - Added ``AVHWDeviceContext`` parameter to ``IFrameSink::onFrame()`` to prepare zero-copy implementation for encoders --- AVQt/AVQt | 1 + AVQt/CMakeLists.txt | 4 +++ AVQt/filter/DecoderDXVA2.cpp | 2 +- AVQt/filter/DecoderMMAL.cpp | 2 +- AVQt/filter/DecoderQSV.cpp | 3 +- AVQt/filter/DecoderVAAPI.cpp | 21 +++++++----- AVQt/filter/EncoderVAAPI.cpp | 50 +++++++++++++++++++++------- AVQt/filter/EncoderVAAPI.h | 4 +-- AVQt/filter/IEncoder.h | 6 +++- AVQt/filter/private/EncoderVAAPI_p.h | 4 +-- AVQt/output/IFrameSink.h | 3 +- AVQt/output/Muxer.cpp | 34 +++++++++++++++++-- AVQt/output/Muxer.h | 12 ++++--- AVQt/output/OpenGLRenderer.cpp | 3 +- AVQt/output/OpenGLRenderer.h | 2 +- AVQt/output/private/Muxer_p.h | 4 ++- Player/main.cpp | 22 ++++++------ 17 files changed, 128 insertions(+), 49 deletions(-) diff --git a/AVQt/AVQt b/AVQt/AVQt index 7e31b52..3acd66f 100644 --- a/AVQt/AVQt +++ b/AVQt/AVQt @@ -14,4 +14,5 @@ #include "output/IAudioSink.h" #include "output/OpenGLRenderer.h" #include "output/OpenALAudioOutput.h" +#include "output/Muxer.h" //#include "output/FrameFileSaver.h" \ No newline at end of file diff --git a/AVQt/CMakeLists.txt b/AVQt/CMakeLists.txt index 436bf39..716663c 100644 --- a/AVQt/CMakeLists.txt +++ b/AVQt/CMakeLists.txt @@ -74,6 +74,10 @@ set(SOURCES output/OpenALAudioOutput.h output/private/OpenALAudioOutput_p.h output/OpenALAudioOutput.cpp + + output/Muxer.h + output/private/Muxer_p.h + output/Muxer.cpp ) add_library(AVQt SHARED ${SOURCES}) diff --git a/AVQt/filter/DecoderDXVA2.cpp b/AVQt/filter/DecoderDXVA2.cpp index 41c5c29..6e2219d 100644 --- a/AVQt/filter/DecoderDXVA2.cpp +++ b/AVQt/filter/DecoderDXVA2.cpp @@ -287,7 +287,7 @@ namespace AVQt { d->m_timebase.num, d->m_timebase.den); QTime time = QTime::currentTime(); - cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0)); + cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0), d->m_pDeviceCtx); qDebug() << "Video CB time:" << time.msecsTo(QTime::currentTime()); av_frame_unref(cbFrame); av_frame_free(&cbFrame); diff --git a/AVQt/filter/DecoderMMAL.cpp b/AVQt/filter/DecoderMMAL.cpp index 2ddf4d6..7a84383 100644 --- a/AVQt/filter/DecoderMMAL.cpp +++ b/AVQt/filter/DecoderMMAL.cpp @@ -292,7 +292,7 @@ namespace AVQt { d->m_timebase.num, d->m_timebase.den); QTime time = QTime::currentTime(); - cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0)); + cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0), nullptr); qDebug() << "Video CB time:" << time.msecsTo(QTime::currentTime()); av_frame_unref(cbFrame); av_frame_free(&cbFrame); diff --git a/AVQt/filter/DecoderQSV.cpp b/AVQt/filter/DecoderQSV.cpp index 464d43c..29de5b4 100644 --- a/AVQt/filter/DecoderQSV.cpp +++ b/AVQt/filter/DecoderQSV.cpp @@ -287,7 +287,8 @@ namespace AVQt { d->m_timebase.num, d->m_timebase.den); QTime time = QTime::currentTime(); - cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0)); + cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0), + av_buffer_ref(d->m_pDeviceCtx)); qDebug() << "Video CB time:" << time.msecsTo(QTime::currentTime()); av_frame_unref(cbFrame); av_frame_free(&cbFrame); diff --git a/AVQt/filter/DecoderVAAPI.cpp b/AVQt/filter/DecoderVAAPI.cpp index d61dcec..919839e 100644 --- a/AVQt/filter/DecoderVAAPI.cpp +++ b/AVQt/filter/DecoderVAAPI.cpp @@ -220,12 +220,12 @@ namespace AVQt { if (d->m_pCodecParams && !d->m_pCodecCtx) { d->m_pCodec = avcodec_find_decoder(d->m_pCodecParams->codec_id); if (!d->m_pCodec) { - qFatal("No audio decoder found"); + qFatal("No video decoder found"); } d->m_pCodecCtx = avcodec_alloc_context3(d->m_pCodec); if (!d->m_pCodecCtx) { - qFatal("Could not allocate audio decoder context, probably out of memory"); + qFatal("Could not allocate video decoder context, probably out of memory"); } ret = av_hwdevice_ctx_create(&d->m_pDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, "/dev/dri/renderD128", nullptr, 0); @@ -267,13 +267,18 @@ namespace AVQt { lock.relock(); } } - AVFrame *frame = av_frame_alloc(); while (true) { + AVFrame *frame = av_frame_alloc(); ret = avcodec_receive_frame(d->m_pCodecCtx, frame); if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) { break; + } else if (ret == -12) { // -12 == Out of memory, didn't know the macro name + av_frame_free(&frame); + msleep(1); + continue; } else if (ret < 0) { - qFatal("%d: Error receiving frame from VAAPI decoder: %s", ret, av_make_error_string(strBuf, strBufSize, ret)); + qFatal("%d: Error receiving frame %d from VAAPI decoder: %s", ret, d->m_pCodecCtx->frame_number, + av_make_error_string(strBuf, strBufSize, ret)); } // auto t1 = NOW(); @@ -295,9 +300,9 @@ namespace AVQt { d->m_timebase.num, d->m_timebase.den); QTime time = QTime::currentTime(); - cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0)); + cb->onFrame(this, cbFrame, static_cast<int64_t>(av_q2d(av_inv_q(d->m_framerate)) * 1000.0), + av_buffer_ref(d->m_pDeviceCtx)); qDebug() << "Video CB time:" << time.msecsTo(QTime::currentTime()); - av_frame_unref(cbFrame); av_frame_free(&cbFrame); // })); } @@ -316,9 +321,9 @@ namespace AVQt { // qDebug("Decoder frame transfer time: %ld us", TIME_US(t1, t3)); // av_frame_free(&swFrame); - av_frame_unref(frame); + av_frame_free(&frame); } - av_frame_free(&frame); +// av_frame_free(&frame); } else { msleep(4); } diff --git a/AVQt/filter/EncoderVAAPI.cpp b/AVQt/filter/EncoderVAAPI.cpp index e94235e..f1f10f0 100644 --- a/AVQt/filter/EncoderVAAPI.cpp +++ b/AVQt/filter/EncoderVAAPI.cpp @@ -8,10 +8,11 @@ #include "output/IPacketSink.h" namespace AVQt { - EncoderVAAPI::EncoderVAAPI(QString encoder, QObject *parent) : QThread(parent), d_ptr(new EncoderVAAPIPrivate(this)) { + EncoderVAAPI::EncoderVAAPI(CODEC codec, int bitrate, QObject *parent) : QThread(parent), d_ptr(new EncoderVAAPIPrivate(this)) { Q_D(AVQt::EncoderVAAPI); - d->m_encoder = std::move(encoder); + d->m_codec = codec; + d->m_bitrate = bitrate; } [[maybe_unused]] EncoderVAAPI::EncoderVAAPI(EncoderVAAPIPrivate &p) : d_ptr(&p) { @@ -34,9 +35,34 @@ namespace AVQt { // constexpr auto strBufSize = 64; // char strBuf[strBufSize]; - d->m_pCodec = avcodec_find_encoder_by_name(d->m_encoder.toLocal8Bit().constData()); + std::string codec_name; + switch (d->m_codec) { + case CODEC::H264: + codec_name = "h264_vaapi"; + break; + case CODEC::HEVC: + codec_name = "hevc_vaapi"; + break; + case CODEC::VP9: + if (qEnvironmentVariable("LIBVA_DRIVER_NAME") == "iHD") { + qFatal("[AVQt::EncoderVAAPI] Unsupported codec: VP9"); + } else { + codec_name = "vp9_vaapi"; + } + break; + case CODEC::VP8: + codec_name = "vp8_vaapi"; + break; + case CODEC::MPEG2: + codec_name = "mpeg2_vaapi"; + break; + case CODEC::AV1: + qFatal("[AVQt::EncoderVAAPI] Unsupported codec: AV1"); + } + + d->m_pCodec = avcodec_find_encoder_by_name(codec_name.c_str()); if (!d->m_pCodec) { - qFatal("Could not find encoder: %s", d->m_encoder.toLocal8Bit().constData()); + qFatal("Could not find encoder: %s", codec_name.c_str()); } return 0; @@ -136,8 +162,8 @@ namespace AVQt { int EncoderVAAPI::init(IFrameSource *source, AVRational framerate, int64_t duration) { Q_UNUSED(source) Q_UNUSED(duration) + Q_UNUSED(framerate) Q_D(AVQt::EncoderVAAPI); - d->m_framerate = framerate; init(); return 0; } @@ -193,7 +219,7 @@ namespace AVQt { } } - void EncoderVAAPI::onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration) { + void EncoderVAAPI::onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration, AVBufferRef *pDeviceCtx) { Q_UNUSED(source) Q_D(AVQt::EncoderVAAPI); @@ -212,7 +238,7 @@ namespace AVQt { break; } - while (d->m_inputQueue.size() > 100) { + while (d->m_inputQueue.size() > 4) { QThread::msleep(1); } { @@ -276,10 +302,10 @@ namespace AVQt { d->m_pCodecCtx->hw_device_ctx = av_buffer_ref(d->m_pDeviceCtx); d->m_pCodecCtx->hw_frames_ctx = av_buffer_ref(d->m_pFramesCtx); -// d->m_pCodecCtx->bit_rate = 5000000; -// d->m_pCodecCtx->rc_min_rate = 4500000; -// d->m_pCodecCtx->rc_max_rate = 6000000; -// d->m_pCodecCtx->rc_buffer_size = 10000000; + d->m_pCodecCtx->bit_rate = d->m_bitrate; + d->m_pCodecCtx->rc_min_rate = static_cast<int>(std::round(d->m_bitrate * 0.8)); + d->m_pCodecCtx->rc_max_rate = static_cast<int>(std::round(d->m_bitrate * 1.1)); + d->m_pCodecCtx->rc_buffer_size = d->m_bitrate * 2; d->m_pCodecCtx->gop_size = 20; d->m_pCodecCtx->max_b_frames = 0; d->m_pCodecCtx->color_primaries = AVCOL_PRI_BT2020; @@ -306,7 +332,7 @@ namespace AVQt { for (const auto &cb: d->m_cbList) { AVCodecParameters *parameters = avcodec_parameters_alloc(); avcodec_parameters_from_context(parameters, d->m_pCodecCtx); - cb->init(this, d->m_framerate, d->m_pCodecCtx->time_base, 0, parameters, nullptr, nullptr); + cb->init(this, av_make_q(0, 1), d->m_pCodecCtx->time_base, 0, parameters, nullptr, nullptr); cb->start(this); } } diff --git a/AVQt/filter/EncoderVAAPI.h b/AVQt/filter/EncoderVAAPI.h index 617160d..6de8187 100644 --- a/AVQt/filter/EncoderVAAPI.h +++ b/AVQt/filter/EncoderVAAPI.h @@ -32,7 +32,7 @@ namespace AVQt { Q_INTERFACES(AVQt::IEncoder) public: - explicit EncoderVAAPI(QString encoder, QObject *parent = nullptr); + explicit EncoderVAAPI(CODEC codec, int bitrate, QObject *parent = nullptr); EncoderVAAPI(EncoderVAAPI &&other) noexcept; @@ -74,7 +74,7 @@ namespace AVQt { Q_INVOKABLE void pause(IFrameSource *source, bool paused) Q_DECL_OVERRIDE; - Q_INVOKABLE void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration) Q_DECL_OVERRIDE; + Q_INVOKABLE void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration, AVBufferRef *pDeviceCtx) Q_DECL_OVERRIDE; signals: diff --git a/AVQt/filter/IEncoder.h b/AVQt/filter/IEncoder.h index 0554f17..9c41a03 100644 --- a/AVQt/filter/IEncoder.h +++ b/AVQt/filter/IEncoder.h @@ -15,6 +15,10 @@ namespace AVQt { Q_INTERFACES(AVQt::IPacketSource) public: + enum class CODEC { + H264, HEVC, VP9, VP8, MPEG2, AV1 + }; + virtual ~IEncoder() = default; virtual bool isPaused() = 0; @@ -31,7 +35,7 @@ namespace AVQt { Q_INVOKABLE virtual void pause(IFrameSource *source, bool pause) = 0; - Q_INVOKABLE virtual void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration) = 0; + Q_INVOKABLE virtual void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration, AVBufferRef *pDeviceCtx) = 0; // IPacketSource diff --git a/AVQt/filter/private/EncoderVAAPI_p.h b/AVQt/filter/private/EncoderVAAPI_p.h index 239bd63..f310119 100644 --- a/AVQt/filter/private/EncoderVAAPI_p.h +++ b/AVQt/filter/private/EncoderVAAPI_p.h @@ -21,9 +21,9 @@ namespace AVQt { EncoderVAAPI *q_ptr; - QString m_encoder{""}; + IEncoder::CODEC m_codec{IEncoder::CODEC::H264}; + int m_bitrate{5 * 1024 * 1024}; - AVRational m_framerate{0, 1}; AVCodec *m_pCodec{nullptr}; AVCodecContext *m_pCodecCtx{nullptr}; AVBufferRef *m_pDeviceCtx{nullptr}, *m_pFramesCtx{nullptr}; diff --git a/AVQt/output/IFrameSink.h b/AVQt/output/IFrameSink.h index 7f36132..bb818f8 100644 --- a/AVQt/output/IFrameSink.h +++ b/AVQt/output/IFrameSink.h @@ -9,6 +9,7 @@ struct AVFrame; struct AVRational; +struct AVBufferRef; namespace AVQt { /*! @@ -89,7 +90,7 @@ namespace AVQt { * @param framerate Source stream framerate * @param duration Source frame presentation duration (inverse of framerate) */ - Q_INVOKABLE virtual void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration) = 0; + Q_INVOKABLE virtual void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration, AVBufferRef *pDeviceCtx) = 0; signals: diff --git a/AVQt/output/Muxer.cpp b/AVQt/output/Muxer.cpp index 6f67887..d612c31 100644 --- a/AVQt/output/Muxer.cpp +++ b/AVQt/output/Muxer.cpp @@ -6,9 +6,10 @@ #include "Muxer.h" namespace AVQt { - Muxer::Muxer(QIODevice *outputDevice, QObject *parent) : QThread(parent), d_ptr(new MuxerPrivate(this)) { + Muxer::Muxer(QIODevice *outputDevice, FORMAT format, QObject *parent) : QThread(parent), d_ptr(new MuxerPrivate(this)) { Q_D(AVQt::Muxer); d->m_outputDevice = outputDevice; + d->m_format = format; } Muxer::Muxer(AVQt::MuxerPrivate &p) : d_ptr(&p) { @@ -59,8 +60,37 @@ namespace AVQt { } else if (!d->m_outputDevice->isWritable()) { qFatal("[AVQt::Muxer] Output device is not writable"); } + + QString outputFormat; + switch (d->m_format) { + case FORMAT::MP4: + if (d->m_outputDevice->isSequential()) { + qFatal("[AVQt::Muxer] MP4 output format is not available on sequential output devices like sockets"); + } + outputFormat = "mp4"; + break; + case FORMAT::MOV: + if (d->m_outputDevice->isSequential()) { + qFatal("[AVQt::Muxer] MOV output format is not available on sequential output devices like sockets"); + } + outputFormat = "mov"; + break; + case FORMAT::MKV: + outputFormat = "matroska"; + break; + case FORMAT::WEBM: + outputFormat = "webm"; + break; + case FORMAT::MPEGTS: + outputFormat = "mpegts"; + break; + case FORMAT::INVALID: + qFatal("[AVQt::Muxer] FORMAT::INVALID is just a placeholder, don't pass it as an argument"); + break; + } + d->m_pFormatContext = avformat_alloc_context(); - d->m_pFormatContext->oformat = av_guess_format("mp4", "", nullptr); + d->m_pFormatContext->oformat = av_guess_format(outputFormat.toLocal8Bit().data(), "", nullptr); d->m_pIOBuffer = static_cast<uint8_t *>(av_malloc(MuxerPrivate::IOBUF_SIZE)); d->m_pIOContext = avio_alloc_context(d->m_pIOBuffer, MuxerPrivate::IOBUF_SIZE, 1, d->m_outputDevice, nullptr, &MuxerPrivate::writeToIO, &MuxerPrivate::seekIO); diff --git a/AVQt/output/Muxer.h b/AVQt/output/Muxer.h index 09d76da..df43944 100644 --- a/AVQt/output/Muxer.h +++ b/AVQt/output/Muxer.h @@ -19,11 +19,12 @@ namespace AVQt { Q_DECLARE_PRIVATE(AVQt::Muxer) - protected: - void run() Q_DECL_OVERRIDE; - public: - explicit Muxer(QIODevice *outputDevice, QObject *parent = nullptr); + enum class FORMAT { + MP4, MOV, MKV, WEBM, MPEGTS, INVALID + }; + + explicit Muxer(QIODevice *outputDevice, FORMAT format, QObject *parent = nullptr); Muxer(Muxer &) = delete; @@ -46,6 +47,9 @@ namespace AVQt { void operator=(const Muxer &) = delete; + protected: + void run() Q_DECL_OVERRIDE; + signals: void started() Q_DECL_OVERRIDE; diff --git a/AVQt/output/OpenGLRenderer.cpp b/AVQt/output/OpenGLRenderer.cpp index d78699b..a314955 100644 --- a/AVQt/output/OpenGLRenderer.cpp +++ b/AVQt/output/OpenGLRenderer.cpp @@ -142,9 +142,10 @@ namespace AVQt { return d->m_paused.load(); } - void OpenGLRenderer::onFrame(IFrameSource *source, AVFrame *frame, int64_t duration) { + void OpenGLRenderer::onFrame(IFrameSource *source, AVFrame *frame, int64_t duration, AVBufferRef *pDeviceCtx) { Q_UNUSED(source) Q_UNUSED(duration) + Q_UNUSED(pDeviceCtx) Q_D(AVQt::OpenGLRenderer); diff --git a/AVQt/output/OpenGLRenderer.h b/AVQt/output/OpenGLRenderer.h index a872577..6fd09bf 100644 --- a/AVQt/output/OpenGLRenderer.h +++ b/AVQt/output/OpenGLRenderer.h @@ -92,7 +92,7 @@ namespace AVQt { * @param timebase Source stream time base, if you don't know what this means, you probably don't want to use it. * @param framerate Source stream framerate */ - Q_INVOKABLE void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration) Q_DECL_OVERRIDE; + Q_INVOKABLE void onFrame(IFrameSource *source, AVFrame *frame, int64_t frameDuration, AVBufferRef *pDeviceCtx) Q_DECL_OVERRIDE; signals: diff --git a/AVQt/output/private/Muxer_p.h b/AVQt/output/private/Muxer_p.h index ca43c35..7aefc08 100644 --- a/AVQt/output/private/Muxer_p.h +++ b/AVQt/output/private/Muxer_p.h @@ -28,8 +28,10 @@ namespace AVQt { QIODevice *m_outputDevice{nullptr}; + Muxer::FORMAT m_format{Muxer::FORMAT::INVALID}; + QMutex m_initMutex{}; - static constexpr size_t IOBUF_SIZE{4 * 1024}; // 4 KB + static constexpr int64_t IOBUF_SIZE{4 * 1024}; // 4 KB uint8_t *m_pIOBuffer{nullptr}; AVIOContext *m_pIOContext{nullptr}; AVFormatContext *m_pFormatContext{nullptr}; diff --git a/Player/main.cpp b/Player/main.cpp index 83656d8..dd72979 100644 --- a/Player/main.cpp +++ b/Player/main.cpp @@ -63,17 +63,17 @@ int main(int argc, char *argv[]) { inputFile->open(QIODevice::ReadWrite); AVQt::Demuxer demuxer(inputFile); -// AVQt::AudioDecoder decoder; -// AVQt::OpenALAudioOutput output; + AVQt::AudioDecoder decoder; + AVQt::OpenALAudioOutput output; -// demuxer.registerCallback(&decoder, AVQt::IPacketSource::CB_AUDIO); -// decoder.registerCallback(&output); + demuxer.registerCallback(&decoder, AVQt::IPacketSource::CB_AUDIO); + decoder.registerCallback(&output); AVQt::IDecoder *videoDecoder; AVQt::IEncoder *videoEncoder; #ifdef Q_OS_LINUX videoDecoder = new AVQt::DecoderVAAPI; - videoEncoder = new AVQt::EncoderVAAPI("hevc_vaapi"); + videoEncoder = new AVQt::EncoderVAAPI(AVQt::IEncoder::CODEC::HEVC, 10 * 1000 * 1000); #elif defined(Q_OS_WINDOWS) videoDecoder = new AVQt::DecoderDXVA2(); #else @@ -82,14 +82,14 @@ int main(int argc, char *argv[]) { AVQt::OpenGLRenderer renderer; demuxer.registerCallback(videoDecoder, AVQt::IPacketSource::CB_VIDEO); -// videoDecoder->registerCallback(videoEncoder); + videoDecoder->registerCallback(videoEncoder); -// QFile outputFile("output.mp4"); -// outputFile.open(QIODevice::ReadWrite | QIODevice::Truncate); -// outputFile.seek(0); -// AVQt::Muxer muxer(&outputFile); + QFile outputFile("output.mp4"); + outputFile.open(QIODevice::ReadWrite | QIODevice::Truncate); + outputFile.seek(0); + AVQt::Muxer muxer(&outputFile, AVQt::Muxer::FORMAT::MP4); -// videoEncoder->registerCallback(&muxer, AVQt::IPacketSource::CB_VIDEO); + videoEncoder->registerCallback(&muxer, AVQt::IPacketSource::CB_VIDEO); videoDecoder->registerCallback(&renderer); renderer.setMinimumSize(QSize(360, 240)); -- GitLab