diff --git a/AVQt/CMakeLists.txt b/AVQt/CMakeLists.txt index 716663c6713417984f3427e66b33abadee736293..279a50548334b636ee620adeff82c56466d61fda 100644 --- a/AVQt/CMakeLists.txt +++ b/AVQt/CMakeLists.txt @@ -183,6 +183,6 @@ if (WIN32) target_link_libraries(AVQt opengl32 OpenAL32) target_link_libraries(AVQtStatic opengl32 OpenAL32) else () - target_link_libraries(AVQt GL openal) - target_link_libraries(AVQtStatic GL openal) + target_link_libraries(AVQt GL openal EGL GLU) + target_link_libraries(AVQtStatic GL openal EGL GLU) endif () \ No newline at end of file diff --git a/AVQt/output/OpenGLRenderer.cpp b/AVQt/output/OpenGLRenderer.cpp index a314955534cc0554bd374627f52feadc9769a353..61c79bc3d0aa1d7a9f8a0791b0599be334b3c0a2 100644 --- a/AVQt/output/OpenGLRenderer.cpp +++ b/AVQt/output/OpenGLRenderer.cpp @@ -6,21 +6,84 @@ #include "OpenGLRenderer.h" #include <QtGui> - +#include <QImage> + +#include <va/va.h> +#include <va/va_x11.h> +#include <va/va_drm.h> +#include <va/va_drmcommon.h> +#include <libdrm/drm_fourcc.h> + +#include <EGL/egl.h> +#include <EGL/eglext.h> +#include <GL/glu.h> +#include <fcntl.h> +#include <cstdio> +#include <unistd.h> + +#pragma clang diagnostic push +#pragma ide diagnostic ignored "HidingNonVirtualFunction" extern "C" { #include <libavutil/frame.h> #include <libavcodec/avcodec.h> #include <libavutil/imgutils.h> #include <libavutil/hwcontext.h> +#include <libavutil/hwcontext_vaapi.h> } +#define LOOKUP_FUNCTION(type, func) \ + type func = (type) eglGetProcAddress(#func); \ + if (!(func)) { qFatal("eglGetProcAddress(" #func ")"); } + static void loadResources() { Q_INIT_RESOURCE(resources); } +static int closefd(int fd) { + return close(fd); +} + +std::string eglErrorString(EGLint error) { + switch (error) { + case EGL_SUCCESS: + return "No error"; + case EGL_NOT_INITIALIZED: + return "EGL not initialized or failed to initialize"; + case EGL_BAD_ACCESS: + return "Resource inaccessible"; + case EGL_BAD_ALLOC: + return "Cannot allocate resources"; + case EGL_BAD_ATTRIBUTE: + return "Unrecognized attribute or attribute value"; + case EGL_BAD_CONTEXT: + return "Invalid EGL context"; + case EGL_BAD_CONFIG: + return "Invalid EGL frame buffer configuration"; + case EGL_BAD_CURRENT_SURFACE: + return "Current surface is no longer valid"; + case EGL_BAD_DISPLAY: + return "Invalid EGL display"; + case EGL_BAD_SURFACE: + return "Invalid surface"; + case EGL_BAD_MATCH: + return "Inconsistent arguments"; + case EGL_BAD_PARAMETER: + return "Invalid argument"; + case EGL_BAD_NATIVE_PIXMAP: + return "Invalid native pixmap"; + case EGL_BAD_NATIVE_WINDOW: + return "Invalid native window"; + case EGL_CONTEXT_LOST: + return "Context lost"; + default: + return "Unknown error " + std::to_string(int(error)); + } +} + namespace AVQt { - OpenGLRenderer::OpenGLRenderer(QWindow *parent) : QOpenGLWindow(QOpenGLWindow::NoPartialUpdate, parent), + OpenGLRenderer::OpenGLRenderer(QWidget *parent) : QOpenGLWidget(parent), d_ptr(new OpenGLRendererPrivate(this)) { + setAttribute(Qt::WA_QuitOnClose); } [[maybe_unused]] [[maybe_unused]] OpenGLRenderer::OpenGLRenderer(OpenGLRendererPrivate &p) : d_ptr(&p) { @@ -65,7 +128,7 @@ namespace AVQt { qDebug("Started renderer"); QMetaObject::invokeMethod(this, "showNormal", Qt::QueuedConnection); - QMetaObject::invokeMethod(this, "requestActivate", Qt::QueuedConnection); +// QMetaObject::invokeMethod(this, "requestActivate", Qt::QueuedConnection); QMetaObject::invokeMethod(this, "update", Qt::QueuedConnection); started(); @@ -143,11 +206,19 @@ namespace AVQt { } void OpenGLRenderer::onFrame(IFrameSource *source, AVFrame *frame, int64_t duration, AVBufferRef *pDeviceCtx) { + Q_D(AVQt::OpenGLRenderer); Q_UNUSED(source) Q_UNUSED(duration) Q_UNUSED(pDeviceCtx) - Q_D(AVQt::OpenGLRenderer); + QMutexLocker onFrameLock{&d->m_onFrameMutex}; + + bool shouldBe = true; + + if (d->m_firstFrame.compare_exchange_strong(shouldBe, false)) { + d->m_pVAContext = static_cast<AVVAAPIDeviceContext *>(reinterpret_cast<AVHWDeviceContext *>(pDeviceCtx->data)->hwctx); + d->m_VADisplay = d->m_pVAContext->display; + } QPair<AVFrame *, int64_t> newFrame; @@ -156,8 +227,8 @@ namespace AVQt { char strBuf[64]; qDebug("Pixel format: %s", av_get_pix_fmt_string(strBuf, 64, static_cast<AVPixelFormat>(frame->format))); switch (frame->format) { - case AV_PIX_FMT_VAAPI: - case AV_PIX_FMT_DRM_PRIME: +// case AV_PIX_FMT_VAAPI: +// case AV_PIX_FMT_DRM_PRIME: case AV_PIX_FMT_DXVA2_VLD: qDebug("Transferring frame from GPU to CPU"); av_hwframe_transfer_data(newFrame.first, frame, 0); @@ -175,7 +246,7 @@ namespace AVQt { //qDebug() << "Pixel format:" << av_get_pix_fmt_string(strBuf, 64, static_cast<AVPixelFormat>(frame->format)); - while (d->m_renderQueue.size() >= 100) { + while (d->m_renderQueue.size() >= 4) { QThread::msleep(4); } @@ -188,6 +259,51 @@ namespace AVQt { loadResources(); + EGLint visual_attr[] = { + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, + EGL_NONE + }; + d->m_EGLDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (d->m_EGLDisplay == EGL_NO_DISPLAY) { + d->m_EGLDisplay = eglGetDisplay(static_cast<EGLNativeDisplayType>(XOpenDisplay(nullptr))); + } + if (!eglInitialize(d->m_EGLDisplay, nullptr, nullptr)) { + qFatal("eglInitialize"); + } + if (!eglBindAPI(EGL_OPENGL_API)) { + qFatal("eglBindAPI"); + } + + EGLConfig cfg; + EGLint cfg_count; + if (!eglChooseConfig(d->m_EGLDisplay, visual_attr, &cfg, 1, &cfg_count) || (cfg_count < 1)) { + qFatal("eglChooseConfig: %s", eglErrorString(eglGetError()).c_str()); + } + EGLint ctx_attr[] = { + EGL_CONTEXT_OPENGL_PROFILE_MASK, + EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, + EGL_CONTEXT_MAJOR_VERSION, 3, + EGL_CONTEXT_MINOR_VERSION, 3, + EGL_CONTEXT_OPENGL_DEBUG, EGL_TRUE, + EGL_NONE + }; + d->m_EGLContext = eglCreateContext(d->m_EGLDisplay, cfg, EGL_NO_CONTEXT, ctx_attr); + if (d->m_EGLContext == EGL_NO_CONTEXT) { + qFatal("eglCreateContext"); + } + + qDebug("EGL Version: %s", eglQueryString(d->m_EGLDisplay, EGL_VERSION)); + +// d->m_EGLSurface = eglCreateWindowSurface(d->m_EGLDisplay, cfg, this->winId(), nullptr); +// if (d->m_EGLSurface == EGL_NO_SURFACE) { +// qFatal("Could not create EGL surface: %s", eglErrorString(eglGetError()).c_str()); +// } + QByteArray shaderVersionString; if (context()->isOpenGLES()) { @@ -221,93 +337,30 @@ namespace AVQt { d->m_program->setUniformValue("textureU", 1); d->m_program->setUniformValue("textureV", 2); d->m_program->release(); + } - float vertices[] = { - 1, 1, 0, // top right - 1, -1, 0, // bottom right - -1, -1, 0, // bottom left - -1, 1, 0 // top left - }; - - float vertTexCoords[] = { - 0, 0, - 1, 1, - 0, 1, - 1, 0 - }; - -// QColor vertexColors[] = { -// QColor(0xf6, 0xa5, 0x09, 128), -// QColor(0xcb, 0x2d, 0xde, 128),av_q2d(av_inv_q(framerate)) -// QColor(0x0e, 0xee, 0xd1, 128), -// QColor(0x06, 0x89, 0x18, 128) -// }; +#pragma clang diagnostic push +#pragma ide diagnostic ignored "modernize-use-auto" - std::vector<float> vertexBufferData(5 * 4); // 8 entries per vertex * 4 vertices + void OpenGLRenderer::paintGL() { + Q_D(AVQt::OpenGLRenderer); +// auto t1 = std::chrono::high_resolution_clock::now(); - float *buf = vertexBufferData.data(); + LOOKUP_FUNCTION(PFNEGLCREATEIMAGEKHRPROC, eglCreateImageKHR) + LOOKUP_FUNCTION(PFNEGLDESTROYIMAGEKHRPROC, eglDestroyImageKHR) + LOOKUP_FUNCTION(PFNGLEGLIMAGETARGETTEXTURE2DOESPROC, glEGLImageTargetTexture2DOES) - for (int v = 0; v < 4; ++v, buf += 5) { - buf[0] = vertices[3 * v]; - buf[1] = vertices[3 * v + 1]; - buf[2] = vertices[3 * v + 2]; - buf[3] = vertTexCoords[v]; - buf[4] = vertTexCoords[v + 1]; + if (d->m_currentFrame) { + int display_width = width(); + int display_height = (width() * d->m_currentFrame->height + d->m_currentFrame->width / 2) / d->m_currentFrame->width; + if (display_height > height()) { + display_width = (height() * d->m_currentFrame->width + d->m_currentFrame->height / 2) / d->m_currentFrame->height; + display_height = height(); + } + glViewport((width() - display_width) / 2, (height() - display_height) / 2, display_width, display_height); } - d->m_vbo = QOpenGLBuffer(QOpenGLBuffer::VertexBuffer); - d->m_vbo.create(); - // QObject::connect(&renderer, &AVQt::OpenGLRenderer::started, [&]() { - d->m_vbo.setUsagePattern(QOpenGLBuffer::StaticDraw); - d->m_vbo.bind(); - - d->m_vbo.allocate(vertexBufferData.data(), static_cast<int>(vertexBufferData.size() * sizeof(float))); - - d->m_vao.create(); - d->m_vao.bind(); - - uint indices[] = { - 0, 1, 3, // first tri - 1, 2, 3 // second tri - }; - - d->m_ibo = QOpenGLBuffer(QOpenGLBuffer::IndexBuffer); - d->m_ibo.create(); - d->m_ibo.setUsagePattern(QOpenGLBuffer::StaticDraw); - d->m_ibo.bind(); - d->m_ibo.allocate(indices, sizeof(indices)); - - - int stride = 5 * sizeof(float); - - // layout location 0 - vec3 with coords - d->m_program->enableAttributeArray(0); - d->m_program->setAttributeBuffer(0, GL_FLOAT, 0, 3, stride); - - // layout location 1 - vec3 with colors -// d->m_program->enableAttributeArray(1); -// int colorOffset = 3 * sizeof(float); -// d->m_program->setAttributeBuffer(1, GL_FLOAT, colorOffset, 3, stride); - - // layout location 1 - vec2 with texture coordinates - d->m_program->enableAttributeArray(1); - int texCoordsOffset = 3 * sizeof(float); - d->m_program->setAttributeBuffer(1, GL_FLOAT, texCoordsOffset, 2, stride); - -// // layout location 2 - int with texture id -// d->m_program->enableAttributeArray(2); -// d->m_program->setAttributeValue(2, d->m_yTexture->textureId()); - - // Release (unbind) all - - d->m_vbo.release(); - d->m_vao.release(); - } - - void OpenGLRenderer::paintGL() { - Q_D(AVQt::OpenGLRenderer); -// auto t1 = std::chrono::high_resolution_clock::now(); // Clear background glClearColor(0.0f, 0.0f, 0.0f, 1.0f); @@ -363,15 +416,7 @@ namespace AVQt { } if (d->m_currentFrame) { - if (d->m_currentFrame->format == AV_PIX_FMT_BGRA) { - av_freep(&d->m_currentFrame->data[0]); - av_frame_free(&d->m_currentFrame); - d->m_currentFrame = nullptr; - } else { - av_frame_unref(d->m_currentFrame); - av_frame_free(&d->m_currentFrame); - d->m_currentFrame = nullptr; - } + av_frame_free(&d->m_currentFrame); } d->m_currentFrame = frame.first; @@ -379,134 +424,333 @@ namespace AVQt { } if (firstFrame) { - bool VTexActive = false, UTexActive = false; - QSize YSize, USize, VSize; - QOpenGLTexture::TextureFormat textureFormatY, textureFormatU, textureFormatV; - QOpenGLTexture::PixelFormat pixelFormatY, pixelFormatU, pixelFormatV; - QOpenGLTexture::PixelType pixelType; - switch (static_cast<AVPixelFormat>(d->m_currentFrame->format)) { - case AV_PIX_FMT_BGRA: - YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); - textureFormatY = QOpenGLTexture::RGBA8_UNorm; - pixelFormatY = QOpenGLTexture::BGRA; - pixelType = QOpenGLTexture::UInt8; + // Frame has 64 pixel alignment, set max height coord to cut off additional pixels + float maxTexHeight = 1.0f; + if (d->m_currentFrame->format == AV_PIX_FMT_VAAPI) { + VASurfaceID vaSurfaceId = reinterpret_cast<uintptr_t>(d->m_currentFrame->data[3]); + VAImage vaImage; + vaDeriveImage(d->m_VADisplay, vaSurfaceId, &vaImage); + maxTexHeight = static_cast<float>(d->m_currentFrame->height * 1.0 / (vaImage.height + 2.0)); + vaDestroyImage(d->m_VADisplay, vaImage.image_id); + } + + float vertices[] = { + 1, 1, 0, // top right + 1, -1, 0, // bottom right + -1, -1, 0, // bottom left + -1, 1, 0 // top left + }; + + float vertTexCoords[] = { + 0, 0, + maxTexHeight, maxTexHeight, + 0, 1, + 1, 0 + }; + + std::vector<float> vertexBufferData(5 * 4); // 8 entries per vertex * 4 vertices + + float *buf = vertexBufferData.data(); + + for (int v = 0; v < 4; ++v, buf += 5) { + buf[0] = vertices[3 * v]; + buf[1] = vertices[3 * v + 1]; + buf[2] = vertices[3 * v + 2]; + + buf[3] = vertTexCoords[v]; + buf[4] = vertTexCoords[v + 1]; + } + + d->m_vbo = QOpenGLBuffer(QOpenGLBuffer::VertexBuffer); + d->m_vbo.create(); + d->m_vbo.setUsagePattern(QOpenGLBuffer::StaticDraw); + d->m_vbo.bind(); + + d->m_vbo.allocate(vertexBufferData.data(), static_cast<int>(vertexBufferData.size() * sizeof(float))); + + d->m_vao.create(); + d->m_vao.bind(); + + uint indices[] = { + 0, 1, 3, // first tri + 1, 2, 3 // second tri + }; + + d->m_ibo = QOpenGLBuffer(QOpenGLBuffer::IndexBuffer); + d->m_ibo.create(); + d->m_ibo.setUsagePattern(QOpenGLBuffer::StaticDraw); + d->m_ibo.bind(); + d->m_ibo.allocate(indices, sizeof(indices)); + + + int stride = 5 * sizeof(float); + + // layout location 0 - vec3 with coords + d->m_program->enableAttributeArray(0); + d->m_program->setAttributeBuffer(0, GL_FLOAT, 0, 3, stride); + + // layout location 1 - vec2 with texture coordinates + d->m_program->enableAttributeArray(1); + int texCoordsOffset = 3 * sizeof(float); + d->m_program->setAttributeBuffer(1, GL_FLOAT, texCoordsOffset, 2, stride); + + // Release (unbind) all + d->m_vbo.release(); + d->m_vao.release(); + if (d->m_currentFrame->format == AV_PIX_FMT_VAAPI) { + glGenTextures(2, d->m_textures); + for (const auto &texture : d->m_textures) { + glBindTexture(GL_TEXTURE_2D, texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + } + d->m_program->bind(); + d->m_program->setUniformValue("inputFormat", 1); + d->m_program->release(); + } else { + bool VTexActive = false, UTexActive = false; + QSize YSize, USize, VSize; + QOpenGLTexture::TextureFormat textureFormatY, textureFormatU, textureFormatV; + QOpenGLTexture::PixelFormat pixelFormatY, pixelFormatU, pixelFormatV; + QOpenGLTexture::PixelType pixelType; + switch (static_cast<AVPixelFormat>(d->m_currentFrame->format)) { + case AV_PIX_FMT_BGRA: + YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); + textureFormatY = QOpenGLTexture::RGBA8_UNorm; + pixelFormatY = QOpenGLTexture::BGRA; + pixelType = QOpenGLTexture::UInt8; + break; + case AV_PIX_FMT_YUV420P: + YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); + USize = VSize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); + textureFormatY = textureFormatU = textureFormatV = QOpenGLTexture::R8_UNorm; + pixelFormatY = pixelFormatU = pixelFormatV = QOpenGLTexture::Red; + pixelType = QOpenGLTexture::UInt8; + UTexActive = VTexActive = true; + break; + case AV_PIX_FMT_YUV420P10: + YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); + USize = VSize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); + textureFormatY = textureFormatU = textureFormatV = QOpenGLTexture::R16_UNorm; + pixelFormatY = pixelFormatU = pixelFormatV = QOpenGLTexture::Red; + pixelType = QOpenGLTexture::UInt16; + UTexActive = VTexActive = true; + break; + case AV_PIX_FMT_NV12: + YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); + USize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); + textureFormatY = QOpenGLTexture::R8_UNorm; + textureFormatU = QOpenGLTexture::RG8_UNorm; + pixelFormatY = QOpenGLTexture::Red; + pixelFormatU = QOpenGLTexture::RG; + pixelType = QOpenGLTexture::UInt8; + UTexActive = true; + break; + case AV_PIX_FMT_P010: + YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); + USize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); + textureFormatY = QOpenGLTexture::R16_UNorm; + textureFormatU = QOpenGLTexture::RG16_UNorm; + pixelFormatY = QOpenGLTexture::Red; + pixelFormatU = QOpenGLTexture::RG; + pixelType = QOpenGLTexture::UInt16; + UTexActive = true; + break; + default: + qFatal("[AVQt::OpenGLRenderer] Unsupported pixel format"); + break; + } + d->m_yTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); + d->m_yTexture->setSize(YSize.width(), YSize.height()); + d->m_yTexture->setFormat(textureFormatY); + d->m_yTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); + d->m_yTexture->allocateStorage(pixelFormatY, pixelType); + if (UTexActive) { + d->m_uTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); + d->m_uTexture->setSize(USize.width(), USize.height()); + d->m_uTexture->setFormat(textureFormatU); + d->m_uTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); + d->m_uTexture->allocateStorage(pixelFormatU, pixelType); + } + if (VTexActive) { + d->m_vTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); + d->m_vTexture->setSize(VSize.width(), VSize.height()); + d->m_vTexture->setFormat(textureFormatV); + d->m_vTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); + d->m_vTexture->allocateStorage(pixelFormatV, pixelType); + } + } + } + + if (d->m_currentFrame->format == AV_PIX_FMT_VAAPI) { + for (int i = 0; i < 2; ++i) { + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_2D, d->m_textures[i]); + eglDestroyImageKHR(d->m_EGLDisplay, d->m_EGLImages[i]); + } + + VASurfaceID va_surface = reinterpret_cast<uintptr_t>(d->m_currentFrame->data[3]); + +// VAImage vaImage; +// vaDeriveImage(d->m_VADisplay, va_surface, &vaImage); +// VABufferInfo vaBufferInfo; +// memset(&vaBufferInfo, 0, sizeof(VABufferInfo)); +// vaBufferInfo.mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_VA; +// vaAcquireBufferHandle(d->m_VADisplay, vaImage.buf, &vaBufferInfo); + + VADRMPRIMESurfaceDescriptor prime; + if (vaExportSurfaceHandle(d->m_VADisplay, va_surface, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS, &prime) != + VA_STATUS_SUCCESS) { + qFatal("[AVQt::OpenGLRenderer] Could not export VA surface handle"); + } + vaSyncSurface(d->m_VADisplay, va_surface); + + static uint32_t formats[2]; + switch (prime.fourcc) { +// switch (vaImage.format.fourcc) { + case VA_FOURCC_P010: + formats[0] = DRM_FORMAT_R16; + formats[1] = DRM_FORMAT_GR1616; break; - case AV_PIX_FMT_YUV420P: - YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); - USize = VSize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); - textureFormatY = textureFormatU = textureFormatV = QOpenGLTexture::R8_UNorm; - pixelFormatY = pixelFormatU = pixelFormatV = QOpenGLTexture::Red; - pixelType = QOpenGLTexture::UInt8; - UTexActive = VTexActive = true; + case VA_FOURCC_NV12: + formats[0] = DRM_FORMAT_R8; + formats[1] = DRM_FORMAT_GR88; break; - case AV_PIX_FMT_YUV420P10: - YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); - USize = VSize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); - textureFormatY = textureFormatU = textureFormatV = QOpenGLTexture::R16_UNorm; - pixelFormatY = pixelFormatU = pixelFormatV = QOpenGLTexture::Red; - pixelType = QOpenGLTexture::UInt16; - UTexActive = VTexActive = true; + default: + qFatal("Unsupported pixel format: %s", av_fourcc2str(prime.fourcc)); +// qFatal("Unsupported pixel format: %s", av_fourcc2str(vaImage.format.fourcc)); + } + + for (int i = 0; i < 2; ++i) { + if (prime.layers[i].drm_format != formats[i]) { + qFatal("[AVQt::OpenGLRenderer] Invalid pixel format: %s", av_fourcc2str(prime.layers[i].drm_format)); + } + +#define LAYER i +#define PLANE 0 + const EGLint *img_attr = new EGLint[]{ + EGL_LINUX_DRM_FOURCC_EXT, static_cast<EGLint>(prime.layers[LAYER].drm_format), + EGL_WIDTH, static_cast<EGLint>(prime.width / (i + 1)), + EGL_HEIGHT, static_cast<EGLint>(prime.height / (i + 1)), + EGL_DMA_BUF_PLANE0_FD_EXT, prime.objects[prime.layers[LAYER].object_index[PLANE]].fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, static_cast<EGLint>(prime.layers[LAYER].offset[PLANE]), + EGL_DMA_BUF_PLANE0_PITCH_EXT, static_cast<EGLint>(prime.layers[LAYER].pitch[PLANE]), + EGL_NONE + }; +// const EGLint *img_attr = new EGLint[]{ +// EGL_WIDTH, vaImage.width, +// EGL_HEIGHT, vaImage.height, +// EGL_LINUX_DRM_FOURCC_EXT, static_cast<EGLint>(formats[i]), +// EGL_DMA_BUF_PLANE0_FD_EXT, static_cast<EGLint>(vaBufferInfo.handle), +// EGL_DMA_BUF_PLANE0_OFFSET_EXT, static_cast<EGLint>(vaImage.offsets[i]), +// EGL_DMA_BUF_PLANE0_PITCH_EXT, static_cast<EGLint>(vaImage.pitches[i]), +// EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, +// EGL_NONE +// }; + while (eglGetError() != EGL_SUCCESS); + d->m_EGLImages[i] = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, + img_attr); + + if (!d->m_EGLImages[i] || d->m_EGLImages[i] == EGL_NO_IMAGE_KHR) { + qDebug("[AVQt::OpenGLRenderer] Could not create %s EGLImage: %s", (i ? "UV" : "Y"), + eglErrorString(eglGetError()).c_str()); + } +#undef LAYER +#undef PLANE + + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_2D, d->m_textures[i]); + while (glGetError() != GL_NO_ERROR); + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, d->m_EGLImages[i]); + auto err = glGetError(); + + if (err != GL_NO_ERROR) { + qFatal("Could not map EGL image to OGL texture: %#0.4x, %s", err, gluErrorString(err)); + } + +// void *data = new uint16_t[prime.width * prime.height]; +// +// glGetTexImage(GL_TEXTURE_2D, 0, GL_RED, GL_UNSIGNED_SHORT, data); +// QImage image(reinterpret_cast<uint8_t *>(data), prime.width, prime.height, QImage::Format_Grayscale16); +// image.save("output.bmp"); + +// exit(0); + } +// vaReleaseBufferHandle(d->m_VADisplay, vaImage.buf); +// vaDestroyImage(d->m_VADisplay, vaImage.image_id); + for (int i = 0; i < (int) prime.num_objects; ++i) { + closefd(prime.objects[i].fd); + } + } else { +// qDebug("Frame duration: %ld ms", d->m_currentFrameTimeout); + if (differentPixFmt) { + d->m_program->bind(); + } + d->m_yTexture->bind(0); + if (d->m_uTexture) { + d->m_uTexture->bind(1); + } + if (d->m_vTexture) { + d->m_vTexture->bind(2); + } + switch (d->m_currentFrame->format) { + case AV_PIX_FMT_BGRA: + d->m_yTexture->setData(QOpenGLTexture::PixelFormat::BGRA, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[0])); + if (differentPixFmt) { + d->m_program->setUniformValue("inputFormat", 0); + } break; case AV_PIX_FMT_NV12: - YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); - USize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); - textureFormatY = QOpenGLTexture::R8_UNorm; - textureFormatU = QOpenGLTexture::RG8_UNorm; - pixelFormatY = QOpenGLTexture::Red; - pixelFormatU = QOpenGLTexture::RG; - pixelType = QOpenGLTexture::UInt8; - UTexActive = true; + d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[0])); + d->m_uTexture->setData(QOpenGLTexture::RG, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[1])); + if (differentPixFmt) { + d->m_program->setUniformValue("inputFormat", 1); + } break; case AV_PIX_FMT_P010: - YSize = QSize(d->m_currentFrame->width, d->m_currentFrame->height); - USize = QSize(d->m_currentFrame->width / 2, d->m_currentFrame->height / 2); - textureFormatY = QOpenGLTexture::R16_UNorm; - textureFormatU = QOpenGLTexture::RG16_UNorm; - pixelFormatY = QOpenGLTexture::Red; - pixelFormatU = QOpenGLTexture::RG; - pixelType = QOpenGLTexture::UInt16; - UTexActive = true; + d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, + const_cast<const uint8_t *>(d->m_currentFrame->data[0])); + d->m_uTexture->setData(QOpenGLTexture::RG, QOpenGLTexture::UInt16, + const_cast<const uint8_t *>(d->m_currentFrame->data[1])); + if (differentPixFmt) { + d->m_program->setUniformValue("inputFormat", 1); + } break; - default: - qFatal("[AVQt::OpenGLRenderer] Unsupported pixel format"); + case AV_PIX_FMT_YUV420P: + d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[0])); + d->m_uTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[1])); + d->m_vTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, + const_cast<const uint8_t *>(d->m_currentFrame->data[2])); + if (differentPixFmt) { + d->m_program->setUniformValue("inputFormat", 2); + } break; + case AV_PIX_FMT_YUV420P10: + d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, + const_cast<const uint8_t *>(d->m_currentFrame->data[0])); + d->m_uTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, + const_cast<const uint8_t *>(d->m_currentFrame->data[1])); + d->m_vTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, + const_cast<const uint8_t *>(d->m_currentFrame->data[2])); + if (differentPixFmt) { + d->m_program->setUniformValue("inputFormat", 3); + } + break; + default: + qFatal("Pixel format not supported"); } - d->m_yTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); - d->m_yTexture->setSize(YSize.width(), YSize.height()); - d->m_yTexture->setFormat(textureFormatY); - d->m_yTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); - d->m_yTexture->allocateStorage(pixelFormatY, pixelType); - if (UTexActive) { - d->m_uTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); - d->m_uTexture->setSize(USize.width(), USize.height()); - d->m_uTexture->setFormat(textureFormatU); - d->m_uTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); - d->m_uTexture->allocateStorage(pixelFormatU, pixelType); - } - if (VTexActive) { - d->m_vTexture = new QOpenGLTexture(QOpenGLTexture::Target2D); - d->m_vTexture->setSize(VSize.width(), VSize.height()); - d->m_vTexture->setFormat(textureFormatV); - d->m_vTexture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); - d->m_vTexture->allocateStorage(pixelFormatV, pixelType); + if (differentPixFmt) { + d->m_program->release(); } } -// qDebug("Frame duration: %ld ms", d->m_currentFrameTimeout); - if (differentPixFmt) { - d->m_program->bind(); - } - d->m_yTexture->bind(0); - if (d->m_uTexture) { - d->m_uTexture->bind(1); - } - if (d->m_vTexture) { - d->m_vTexture->bind(2); - } - switch (d->m_currentFrame->format) { - case AV_PIX_FMT_BGRA: - d->m_yTexture->setData(QOpenGLTexture::PixelFormat::BGRA, QOpenGLTexture::UInt8, - d->m_currentFrame->data[0]); - if (differentPixFmt) { - d->m_program->setUniformValue("inputFormat", 0); - } - break; - case AV_PIX_FMT_NV12: - d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, d->m_currentFrame->data[0]); - d->m_uTexture->setData(QOpenGLTexture::RG, QOpenGLTexture::UInt8, d->m_currentFrame->data[1]); - if (differentPixFmt) { - d->m_program->setUniformValue("inputFormat", 1); - } - break; - case AV_PIX_FMT_P010: - d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, d->m_currentFrame->data[0]); - d->m_uTexture->setData(QOpenGLTexture::RG, QOpenGLTexture::UInt16, d->m_currentFrame->data[1]); - if (differentPixFmt) { - d->m_program->setUniformValue("inputFormat", 1); - } - break; - case AV_PIX_FMT_YUV420P: - d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, d->m_currentFrame->data[0]); - d->m_uTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, d->m_currentFrame->data[1]); - d->m_vTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt8, d->m_currentFrame->data[2]); - if (differentPixFmt) { - d->m_program->setUniformValue("inputFormat", 2); - } - break; - case AV_PIX_FMT_YUV420P10: - // TODO: Fix crappy and low-res colors with 10bit YUV420P10LE, caused by data in lsb - d->m_yTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, d->m_currentFrame->data[0]); - d->m_uTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, d->m_currentFrame->data[1]); - d->m_vTexture->setData(QOpenGLTexture::Red, QOpenGLTexture::UInt16, d->m_currentFrame->data[2]); - if (differentPixFmt) { - d->m_program->setUniformValue("inputFormat", 3); - } - break; - default: - qFatal("Pixel format not supported"); - } - if (differentPixFmt) { - d->m_program->release(); - } } } } else if (d->m_clock) { @@ -526,13 +770,20 @@ namespace AVQt { if (d->m_currentFrame) { qDebug("Drawing frame with PTS: %lld", static_cast<long long>(d->m_currentFrame->pts)); d->m_program->bind(); - if (!d->m_yTexture->isBound(0)) { - d->m_yTexture->bind(0); - if (d->m_uTexture) { - d->m_uTexture->bind(1); - } - if (d->m_vTexture) { - d->m_vTexture->bind(2); + if (d->m_currentFrame->format == AV_PIX_FMT_VAAPI) { + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, d->m_textures[0]); + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, d->m_textures[1]); + } else { + if (!d->m_yTexture->isBound(0)) { + d->m_yTexture->bind(0); + if (d->m_uTexture) { + d->m_uTexture->bind(1); + } + if (d->m_vTexture) { + d->m_vTexture->bind(2); + } } } @@ -548,12 +799,16 @@ namespace AVQt { d->m_vao.release(); d->m_vbo.release(); d->m_program->release(); - d->m_yTexture->release(0); - if (d->m_uTexture) { - d->m_uTexture->release(1); - } - if (d->m_vTexture) { - d->m_vTexture->release(2); + if (d->m_currentFrame->format == AV_PIX_FMT_VAAPI) { + + } else { + d->m_yTexture->release(0); + if (d->m_uTexture) { + d->m_uTexture->release(1); + } + if (d->m_vTexture) { + d->m_vTexture->release(2); + } } } @@ -598,12 +853,19 @@ namespace AVQt { } } +#pragma clang diagnostic pop + void OpenGLRenderer::mouseReleaseEvent(QMouseEvent *event) { if (event->button() == Qt::LeftButton) { pause(nullptr, !isPaused()); } } + void OpenGLRenderer::closeEvent(QCloseEvent *event) { + QApplication::quit(); + QWidget::closeEvent(event); + } + RenderClock *OpenGLRenderer::getClock() { Q_D(AVQt::OpenGLRenderer); return d->m_clock; @@ -635,7 +897,7 @@ namespace AVQt { } void OpenGLRendererPrivate::transformPoint(GLdouble *out, const GLdouble *m, const GLdouble *in) { -#define M(row, col) m[col*4+row] +#define M(row, col) m[(col) * 4 + (row)] out[0] = M(0, 0) * in[0] + M(0, 1) * in[1] + M(0, 2) * in[2] + M(0, 3) * in[3]; out[1] = M(1, 0) * in[0] + M(1, 1) * in[1] + M(1, 2) * in[2] + M(1, 3) * in[3]; out[2] = M(2, 0) * in[0] + M(2, 1) * in[1] + M(2, 2) * in[2] + M(2, 3) * in[3]; @@ -650,4 +912,5 @@ namespace AVQt { int h = static_cast<int>(ts / 1000 / 60 / 60); return QTime(h, m, s, ms); } -} \ No newline at end of file +} +#pragma clang diagnostic pop \ No newline at end of file diff --git a/AVQt/output/OpenGLRenderer.h b/AVQt/output/OpenGLRenderer.h index 6fd09bf5ef1bc8461edc5a5a4fbf0e890e7ab6ba..5e367657d510e416f12716c2ec41a4fec22eb08c 100644 --- a/AVQt/output/OpenGLRenderer.h +++ b/AVQt/output/OpenGLRenderer.h @@ -7,6 +7,8 @@ #include <QtOpenGL> #include <input/IFrameSource.h> +#pragma clang diagnostic push +#pragma ide diagnostic ignored "HidingNonVirtualFunction" extern "C" { #include <libavutil/rational.h> } @@ -20,14 +22,14 @@ namespace AVQt { class RenderClock; - class OpenGLRenderer : public QOpenGLWindow, public IFrameSink { + class OpenGLRenderer : public QOpenGLWidget, public IFrameSink { Q_OBJECT // Q_INTERFACES(AVQt::IFrameSink) Q_DECLARE_PRIVATE(AVQt::OpenGLRenderer) public: - explicit OpenGLRenderer(QWindow *parent = nullptr); + explicit OpenGLRenderer(QWidget *parent = nullptr); OpenGLRenderer(OpenGLRenderer &&other) noexcept; @@ -35,7 +37,7 @@ namespace AVQt { void operator=(const OpenGLRenderer &) = delete; - ~OpenGLRenderer() noexcept; + ~OpenGLRenderer() noexcept Q_DECL_OVERRIDE; bool isPaused() Q_DECL_OVERRIDE; @@ -111,9 +113,12 @@ namespace AVQt { void mouseReleaseEvent(QMouseEvent *event) Q_DECL_OVERRIDE; + void closeEvent(QCloseEvent *event) Q_DECL_OVERRIDE; + OpenGLRendererPrivate *d_ptr; }; } -#endif //LIBAVQT_OPENGLRENDERER_H \ No newline at end of file +#endif //LIBAVQT_OPENGLRENDERER_H +#pragma clang diagnostic pop \ No newline at end of file diff --git a/AVQt/output/private/OpenGLRenderer_p.h b/AVQt/output/private/OpenGLRenderer_p.h index 90a33dbb94c47b4376c6b5261452abcb876190b9..ab5f4ab02f9ba17f818de53561fd7e612c594f32 100644 --- a/AVQt/output/private/OpenGLRenderer_p.h +++ b/AVQt/output/private/OpenGLRenderer_p.h @@ -2,11 +2,23 @@ * \private * \internal */ -#include <QtCore> -#include <QtOpenGL> #include "../RenderClock.h" #include "../OpenGLRenderer.h" +#include <QtCore> +#include <QtOpenGL> + +#include <va/va.h> +#include <va/va_x11.h> +#include <va/va_drmcommon.h> + +#include <EGL/egl.h> +#include <EGL/eglext.h> + +extern "C" { +#include <libavutil/hwcontext_vaapi.h> +} + #ifndef LIBAVQT_OPENGLRENDERER_P_H #define LIBAVQT_OPENGLRENDERER_P_H @@ -42,6 +54,7 @@ namespace AVQt { OpenGLRenderer *q_ptr{nullptr}; + QMutex m_onFrameMutex{}; QMutex m_renderQueueMutex{}; QQueue<QPair<AVFrame *, int64_t>> m_renderQueue{}; @@ -56,6 +69,8 @@ namespace AVQt { QMutex m_currentFrameMutex{}; AVFrame *m_currentFrame{nullptr}; + //OpenGL stuff + QOpenGLVertexArrayObject m_vao{}; QOpenGLBuffer m_vbo{}, m_ibo{}; QOpenGLShaderProgram *m_program{nullptr}; @@ -64,6 +79,15 @@ namespace AVQt { static constexpr uint PROGRAM_VERTEX_ATTRIBUTE{0}; static constexpr uint PROGRAM_TEXCOORD_ATTRIBUTE{1}; + // VAAPI stuff + VADisplay m_VADisplay{nullptr}; + AVVAAPIDeviceContext *m_pVAContext{nullptr}; + EGLDisplay m_EGLDisplay{nullptr}; + EGLSurface m_EGLSurface{nullptr}; + EGLContext m_EGLContext{nullptr}; + EGLImage m_EGLImages[2]{}; + GLuint m_textures[2]{}; + friend class OpenGLRenderer; }; } diff --git a/Player/CMakeLists.txt b/Player/CMakeLists.txt index 7b5e0e2d7235b717ebc42b1edd318b4aa02de322..b5d6e05ad8b84e062f19760ccd82fb06c7290f95 100644 --- a/Player/CMakeLists.txt +++ b/Player/CMakeLists.txt @@ -16,7 +16,7 @@ add_executable(Player main.cpp) target_link_libraries(Player Qt5::Core Qt5::Gui Qt5::Concurrent Qt5::Widgets Qt5::OpenGL avformat avfilter avutil avcodec avdevice swscale swresample AVQt) if (LINUX) - target_link_libraries(Player openal GL) + target_link_libraries(Player openal GL EGL GLU) elseif (WIN32) target_link_libraries(Player OpenAL32 opengl32) -endif () +endif () \ No newline at end of file diff --git a/Player/main.cpp b/Player/main.cpp index dd7297924e2238bda4ed89ca61833b5e06c30c7b..701a212f4cb09d82164e6b1215c18a2a174a349f 100644 --- a/Player/main.cpp +++ b/Player/main.cpp @@ -9,7 +9,7 @@ constexpr auto LOGFILE_LOCATION = "libAVQt.log"; QApplication *app = nullptr; -std::chrono::time_point<std::chrono::system_clock> start; +std::chrono::time_point<std::chrono::system_clock> start; // NOLINT(cert-err58-cpp) void signalHandler(int sigNum) { Q_UNUSED(sigNum) @@ -40,6 +40,7 @@ void messageHandler(QtMsgType type, const QMessageLogContext &context, const QSt } int main(int argc, char *argv[]) { + QGuiApplication::setAttribute(Qt::AA_UseOpenGLES); app = new QApplication(argc, argv); signal(SIGINT, &signalHandler); signal(SIGTERM, &signalHandler); @@ -82,7 +83,7 @@ int main(int argc, char *argv[]) { AVQt::OpenGLRenderer renderer; demuxer.registerCallback(videoDecoder, AVQt::IPacketSource::CB_VIDEO); - videoDecoder->registerCallback(videoEncoder); +// videoDecoder->registerCallback(videoEncoder); QFile outputFile("output.mp4"); outputFile.open(QIODevice::ReadWrite | QIODevice::Truncate); @@ -109,7 +110,7 @@ int main(int argc, char *argv[]) { QObject::connect(app, &QApplication::aboutToQuit, [&] { demuxer.deinit(); -// delete videoEncoder; + delete videoEncoder; delete videoDecoder; });