From b32bc2026a32f1dea2688b619f1547781da6bdaa Mon Sep 17 00:00:00 2001
From: Jonas Karlman <jonas@kwiboo.se>
Date: Tue, 6 Aug 2024 09:06:00 +0000
Subject: [PATCH 01/11] avutil/hwcontext: Add hwdevice type for V4L2 Request
 API

Add a hwdevice type for V4L2 Request API with transfer_data_from support
for AV_PIX_FMT_DRM_PRIME, based on AV_HWDEVICE_TYPE_DRM.

AVV4L2RequestDeviceContext.media_fd can be set by the application or a
media device path can be supplied when hwdevice is created. When none
is supplied it default to -1 and hwaccel will auto-detect a media device
with a capable video device.

Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                         |   7 +
 libavutil/Makefile                |   3 +
 libavutil/hwcontext.c             |   4 +
 libavutil/hwcontext.h             |   1 +
 libavutil/hwcontext_internal.h    |   1 +
 libavutil/hwcontext_v4l2request.c | 261 ++++++++++++++++++++++++++++++
 libavutil/hwcontext_v4l2request.h |  41 +++++
 7 files changed, 318 insertions(+)
 create mode 100644 libavutil/hwcontext_v4l2request.c
 create mode 100644 libavutil/hwcontext_v4l2request.h

diff --git a/configure b/configure
index d77a55b653..b8f82f0fbf 100755
--- a/configure
+++ b/configure
@@ -359,6 +359,7 @@ External library support:
   --enable-omx-rpi         enable OpenMAX IL code for Raspberry Pi [no]
   --enable-rkmpp           enable Rockchip Media Process Platform code [no]
   --disable-v4l2-m2m       disable V4L2 mem2mem code [autodetect]
+  --enable-v4l2-request    enable V4L2 Request API code [no]
   --disable-vaapi          disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
   --disable-vdpau          disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
   --disable-videotoolbox   disable VideoToolbox code [autodetect]
@@ -2025,6 +2026,7 @@ HWACCEL_LIBRARY_LIST="
     mmal
     omx
     opencl
+    v4l2_request
 "
 
 DOCUMENT_LIST="
@@ -3146,6 +3148,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
 ffnvcodec_deps_any="libdl LoadLibrary"
 mediacodec_deps="android mediandk"
 nvdec_deps="ffnvcodec"
+v4l2_request_deps="linux_media_h v4l2_timeval_to_ns"
 vaapi_x11_deps="xlib_x11"
 videotoolbox_hwaccel_deps="videotoolbox pthreads"
 videotoolbox_hwaccel_extralibs="-framework QuartzCore"
@@ -7197,6 +7200,10 @@ if enabled v4l2_m2m; then
     check_cc vp9_v4l2_m2m linux/videodev2.h "int i = V4L2_PIX_FMT_VP9;"
 fi
 
+if enabled v4l2_request; then
+    check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
+fi
+
 check_headers sys/videoio.h
 test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete
 
diff --git a/libavutil/Makefile b/libavutil/Makefile
index 6e6fa8d800..1ce46157dd 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -48,6 +48,7 @@ HEADERS = adler32.h                                                     \
           hwcontext_qsv.h                                               \
           hwcontext_mediacodec.h                                        \
           hwcontext_opencl.h                                            \
+          hwcontext_v4l2request.h                                       \
           hwcontext_vaapi.h                                             \
           hwcontext_videotoolbox.h                                      \
           hwcontext_vdpau.h                                             \
@@ -201,6 +202,7 @@ OBJS-$(CONFIG_MACOS_KPERF)              += macos_kperf.o
 OBJS-$(CONFIG_MEDIACODEC)               += hwcontext_mediacodec.o
 OBJS-$(CONFIG_OPENCL)                   += hwcontext_opencl.o
 OBJS-$(CONFIG_QSV)                      += hwcontext_qsv.o
+OBJS-$(CONFIG_V4L2_REQUEST)             += hwcontext_v4l2request.o
 OBJS-$(CONFIG_VAAPI)                    += hwcontext_vaapi.o
 OBJS-$(CONFIG_VIDEOTOOLBOX)             += hwcontext_videotoolbox.o
 OBJS-$(CONFIG_VDPAU)                    += hwcontext_vdpau.o
@@ -222,6 +224,7 @@ SKIPHEADERS-$(CONFIG_D3D12VA)          += hwcontext_d3d12va.h
 SKIPHEADERS-$(CONFIG_DXVA2)            += hwcontext_dxva2.h
 SKIPHEADERS-$(CONFIG_QSV)              += hwcontext_qsv.h
 SKIPHEADERS-$(CONFIG_OPENCL)           += hwcontext_opencl.h
+SKIPHEADERS-$(CONFIG_V4L2_REQUEST)     += hwcontext_v4l2request.h
 SKIPHEADERS-$(CONFIG_VAAPI)            += hwcontext_vaapi.h
 SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += hwcontext_videotoolbox.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += hwcontext_vdpau.h
diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index fa99a0d8a4..7fae9381da 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -65,6 +65,9 @@ static const HWContextType * const hw_table[] = {
 #endif
 #if CONFIG_VULKAN
     &ff_hwcontext_type_vulkan,
+#endif
+#if CONFIG_V4L2_REQUEST
+    &ff_hwcontext_type_v4l2request,
 #endif
     NULL,
 };
@@ -77,6 +80,7 @@ static const char *const hw_type_names[] = {
     [AV_HWDEVICE_TYPE_D3D12VA] = "d3d12va",
     [AV_HWDEVICE_TYPE_OPENCL] = "opencl",
     [AV_HWDEVICE_TYPE_QSV]    = "qsv",
+    [AV_HWDEVICE_TYPE_V4L2REQUEST] = "v4l2request",
     [AV_HWDEVICE_TYPE_VAAPI]  = "vaapi",
     [AV_HWDEVICE_TYPE_VDPAU]  = "vdpau",
     [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
index bac30debae..8cf50ddbd0 100644
--- a/libavutil/hwcontext.h
+++ b/libavutil/hwcontext.h
@@ -38,6 +38,7 @@ enum AVHWDeviceType {
     AV_HWDEVICE_TYPE_MEDIACODEC,
     AV_HWDEVICE_TYPE_VULKAN,
     AV_HWDEVICE_TYPE_D3D12VA,
+    AV_HWDEVICE_TYPE_V4L2REQUEST,
 };
 
 /**
diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
index e32b786238..fd0cf29c6e 100644
--- a/libavutil/hwcontext_internal.h
+++ b/libavutil/hwcontext_internal.h
@@ -158,6 +158,7 @@ extern const HWContextType ff_hwcontext_type_drm;
 extern const HWContextType ff_hwcontext_type_dxva2;
 extern const HWContextType ff_hwcontext_type_opencl;
 extern const HWContextType ff_hwcontext_type_qsv;
+extern const HWContextType ff_hwcontext_type_v4l2request;
 extern const HWContextType ff_hwcontext_type_vaapi;
 extern const HWContextType ff_hwcontext_type_vdpau;
 extern const HWContextType ff_hwcontext_type_videotoolbox;
diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c
new file mode 100644
index 0000000000..833fbf9f40
--- /dev/null
+++ b/libavutil/hwcontext_v4l2request.c
@@ -0,0 +1,261 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <linux/dma-buf.h>
+#include <linux/media.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "avassert.h"
+#include "hwcontext_drm.h"
+#include "hwcontext_internal.h"
+#include "hwcontext_v4l2request.h"
+#include "mem.h"
+
+static void v4l2request_device_free(AVHWDeviceContext *hwdev)
+{
+    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
+
+    if (hwctx->media_fd >= 0) {
+        close(hwctx->media_fd);
+        hwctx->media_fd = -1;
+    }
+}
+
+static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device,
+                                     AVDictionary *opts, int flags)
+{
+    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
+
+    hwctx->media_fd = -1;
+    hwdev->free = v4l2request_device_free;
+
+    // Use auto-detect
+    if (!device || !device[0])
+        return 0;
+
+    hwctx->media_fd = open(device, O_RDWR);
+    if (hwctx->media_fd < 0)
+        return AVERROR(errno);
+
+    return 0;
+}
+
+static int v4l2request_device_init(AVHWDeviceContext *hwdev)
+{
+    AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
+    struct media_device_info device_info;
+
+    // Use auto-detect
+    if (hwctx->media_fd < 0)
+        return 0;
+
+    if (ioctl(hwctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0)
+        return AVERROR(errno);
+
+    av_log(hwdev, AV_LOG_VERBOSE, "Using V4L2 media driver %s (%u.%u.%u)\n",
+           device_info.driver,
+           device_info.driver_version >> 16,
+           (device_info.driver_version >> 8) & 0xff,
+           device_info.driver_version & 0xff);
+
+    return 0;
+}
+
+static int v4l2request_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+    frame->buf[0] = av_buffer_pool_get(hwfc->pool);
+    if (!frame->buf[0])
+        return AVERROR(ENOMEM);
+
+    frame->data[0] = (uint8_t *)frame->buf[0]->data;
+
+    frame->format = AV_PIX_FMT_DRM_PRIME;
+    frame->width  = hwfc->width;
+    frame->height = hwfc->height;
+
+    return 0;
+}
+
+typedef struct DRMMapping {
+    // Address and length of each mmap()ed region.
+    int nb_regions;
+    int object[AV_DRM_MAX_PLANES];
+    void *address[AV_DRM_MAX_PLANES];
+    size_t length[AV_DRM_MAX_PLANES];
+} DRMMapping;
+
+static void v4l2request_unmap_frame(AVHWFramesContext *hwfc,
+                                    HWMapDescriptor *hwmap)
+{
+    DRMMapping *map = hwmap->priv;
+
+    for (int i = 0; i < map->nb_regions; i++) {
+        struct dma_buf_sync sync = {
+            .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ,
+        };
+        ioctl(map->object[i], DMA_BUF_IOCTL_SYNC, &sync);
+        munmap(map->address[i], map->length[i]);
+    }
+
+    av_free(map);
+}
+
+static int v4l2request_map_frame(AVHWFramesContext *hwfc,
+                                 AVFrame *dst, const AVFrame *src)
+{
+    const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
+    struct dma_buf_sync sync = {
+        .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ,
+    };
+    DRMMapping *map;
+    int ret, i, p, plane;
+    void *addr;
+
+    map = av_mallocz(sizeof(*map));
+    if (!map)
+        return AVERROR(ENOMEM);
+
+    av_assert0(desc->nb_objects <= AV_DRM_MAX_PLANES);
+    for (i = 0; i < desc->nb_objects; i++) {
+        addr = mmap(NULL, desc->objects[i].size, AV_HWFRAME_MAP_READ, MAP_SHARED,
+                    desc->objects[i].fd, 0);
+        if (addr == MAP_FAILED) {
+            av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to memory: %s (%d)\n",
+                   desc->objects[i].fd, strerror(errno), errno);
+            ret = AVERROR(errno);
+            goto fail;
+        }
+
+        map->address[i] = addr;
+        map->length[i]  = desc->objects[i].size;
+        map->object[i]  = desc->objects[i].fd;
+
+        /*
+         * We're not checking for errors here because the kernel may not
+         * support the ioctl, in which case its okay to carry on
+         */
+        ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_SYNC, &sync);
+    }
+    map->nb_regions = i;
+
+    plane = 0;
+    for (i = 0; i < desc->nb_layers; i++) {
+        const AVDRMLayerDescriptor *layer = &desc->layers[i];
+        for (p = 0; p < layer->nb_planes; p++) {
+            dst->data[plane] =
+                (uint8_t *)map->address[layer->planes[p].object_index] +
+                                        layer->planes[p].offset;
+            dst->linesize[plane] =      layer->planes[p].pitch;
+            ++plane;
+        }
+    }
+    av_assert0(plane <= AV_DRM_MAX_PLANES);
+
+    dst->width  = src->width;
+    dst->height = src->height;
+
+    ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
+                                v4l2request_unmap_frame, map);
+    if (ret < 0)
+        goto fail;
+
+    return 0;
+
+fail:
+    for (i = 0; i < desc->nb_objects; i++) {
+        if (map->address[i])
+            munmap(map->address[i], map->length[i]);
+    }
+    av_free(map);
+    return ret;
+}
+
+static int v4l2request_transfer_get_formats(AVHWFramesContext *hwfc,
+                                            enum AVHWFrameTransferDirection dir,
+                                            enum AVPixelFormat **formats)
+{
+    enum AVPixelFormat *pix_fmts;
+
+    if (dir == AV_HWFRAME_TRANSFER_DIRECTION_TO)
+        return AVERROR(ENOSYS);
+
+    pix_fmts = av_malloc_array(2, sizeof(*pix_fmts));
+    if (!pix_fmts)
+        return AVERROR(ENOMEM);
+
+    pix_fmts[0] = hwfc->sw_format;
+    pix_fmts[1] = AV_PIX_FMT_NONE;
+
+    *formats = pix_fmts;
+    return 0;
+}
+
+static int v4l2request_transfer_data_from(AVHWFramesContext *hwfc,
+                                          AVFrame *dst, const AVFrame *src)
+{
+    AVFrame *map;
+    int ret;
+
+    if (dst->width > hwfc->width || dst->height > hwfc->height)
+        return AVERROR(EINVAL);
+
+    map = av_frame_alloc();
+    if (!map)
+        return AVERROR(ENOMEM);
+    map->format = dst->format;
+
+    ret = v4l2request_map_frame(hwfc, map, src);
+    if (ret)
+        goto fail;
+
+    map->width  = dst->width;
+    map->height = dst->height;
+
+    ret = av_frame_copy(dst, map);
+    if (ret)
+        goto fail;
+
+    ret = 0;
+fail:
+    av_frame_free(&map);
+    return ret;
+}
+
+const HWContextType ff_hwcontext_type_v4l2request = {
+    .type                   = AV_HWDEVICE_TYPE_V4L2REQUEST,
+    .name                   = "V4L2 Request API",
+
+    .device_hwctx_size      = sizeof(AVV4L2RequestDeviceContext),
+    .device_create          = v4l2request_device_create,
+    .device_init            = v4l2request_device_init,
+
+    .frames_get_buffer      = v4l2request_get_buffer,
+
+    .transfer_get_formats   = v4l2request_transfer_get_formats,
+    .transfer_data_from     = v4l2request_transfer_data_from,
+
+    .pix_fmts = (const enum AVPixelFormat[]) {
+        AV_PIX_FMT_DRM_PRIME,
+        AV_PIX_FMT_NONE
+    },
+};
diff --git a/libavutil/hwcontext_v4l2request.h b/libavutil/hwcontext_v4l2request.h
new file mode 100644
index 0000000000..0fe42f97b4
--- /dev/null
+++ b/libavutil/hwcontext_v4l2request.h
@@ -0,0 +1,41 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_H
+#define AVUTIL_HWCONTEXT_V4L2REQUEST_H
+
+/**
+ * @file
+ * An API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST.
+ */
+
+/**
+ * V4L2 Request API device details.
+ *
+ * Allocated as AVHWDeviceContext.hwctx
+ */
+typedef struct AVV4L2RequestDeviceContext {
+    /**
+     * File descriptor of media device.
+     *
+     * Defaults to -1 for auto-detect.
+     */
+    int media_fd;
+} AVV4L2RequestDeviceContext;
+
+#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_H */

From 8b12b1ca49f86dee00a68d819990a2d63150ac1a Mon Sep 17 00:00:00 2001
From: Jonas Karlman <jonas@kwiboo.se>
Date: Tue, 6 Aug 2024 09:06:01 +0000
Subject: [PATCH 02/11] avcodec: Add common V4L2 Request API code

Add common helpers for supporting V4L2 Request API hwaccels.

Basic flow for initialization follow the kernel Memory-to-memory
Stateless Video Decoder Interface > Initialization [1].

In init video devices is probed and when a capable device is found it is
initialized for decoding. Codec specific CONTROLs may be set to assist
kernel driver. A supported CAPTURE format is selected. A hw frame ctx is
created and CAPTURE buffers is allocated. OUTPUT buffers and request
objects for a circular queue is also allocated.

In frame_params a buffer pool is created and configured to allocate
CAPTURE buffers.

In flush all queued OUTPUT and CAPTURE buffers is dequeued.

In uninit any pending request is flushed, also video and media device is
closed. The media device is not closed when ownership of the media_fd
has been transfered to the hwdevice.

[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization

Co-developed-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 MAINTAINERS                        |   1 +
 libavcodec/Makefile                |   1 +
 libavcodec/v4l2_request.c          | 441 +++++++++++++++++++++++++++++
 libavcodec/v4l2_request.h          |  84 ++++++
 libavcodec/v4l2_request_internal.h |  42 +++
 5 files changed, 569 insertions(+)
 create mode 100644 libavcodec/v4l2_request.c
 create mode 100644 libavcodec/v4l2_request.h
 create mode 100644 libavcodec/v4l2_request_internal.h

diff --git a/MAINTAINERS b/MAINTAINERS
index 76651d5ff8..391acfbe16 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -278,6 +278,7 @@ Hardware acceleration:
   d3d11va*                              Steve Lhomme
   d3d12va_encode*                       Tong Wu
   mediacodec*                           Matthieu Bouron, Aman Gupta, Zhao Zhili
+  v4l2_request*                         Jonas Karlman (CC jonas@kwiboo.se)
   vaapi*                                Haihao Xiang
   vaapi_encode*                         Mark Thompson, Haihao Xiang
   vdpau*                                Philip Langdale, Carl Eugen Hoyos
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index a4fcce3b42..af60923827 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -176,6 +176,7 @@ OBJS-$(CONFIG_VP3DSP)                  += vp3dsp.o
 OBJS-$(CONFIG_VP56DSP)                 += vp56dsp.o
 OBJS-$(CONFIG_VP8DSP)                  += vp8dsp.o
 OBJS-$(CONFIG_V4L2_M2M)                += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
+OBJS-$(CONFIG_V4L2_REQUEST)            += v4l2_request.o
 OBJS-$(CONFIG_WMA_FREQS)               += wma_freqs.o
 OBJS-$(CONFIG_WMV2DSP)                 += wmv2dsp.o
 
diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c
new file mode 100644
index 0000000000..436c8de4a4
--- /dev/null
+++ b/libavcodec/v4l2_request.c
@@ -0,0 +1,441 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "libavutil/hwcontext_v4l2request.h"
+#include "libavutil/mem.h"
+#include "decode.h"
+#include "v4l2_request_internal.h"
+
+static const AVClass v4l2_request_context_class = {
+    .class_name = "V4L2RequestContext",
+    .item_name  = av_default_item_name,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+int ff_v4l2_request_query_control(AVCodecContext *avctx,
+                                  struct v4l2_query_ext_ctrl *control)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+
+    if (ioctl(ctx->video_fd, VIDIOC_QUERY_EXT_CTRL, control) < 0) {
+        // Skip error logging when driver does not support control id (EINVAL)
+        if (errno != EINVAL) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to query control %u: %s (%d)\n",
+                   control->id, strerror(errno), errno);
+        }
+        return AVERROR(errno);
+    }
+
+    return 0;
+}
+
+int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx,
+                                                uint32_t id)
+{
+    struct v4l2_query_ext_ctrl control = {
+        .id = id,
+    };
+    int ret;
+
+    ret = ff_v4l2_request_query_control(avctx, &control);
+    if (ret < 0)
+        return ret;
+
+    return control.default_value;
+}
+
+int ff_v4l2_request_set_request_controls(V4L2RequestContext *ctx, int request_fd,
+                                         struct v4l2_ext_control *control, int count)
+{
+    struct v4l2_ext_controls controls = {
+        .controls = control,
+        .count = count,
+        .request_fd = request_fd,
+        .which = (request_fd >= 0) ? V4L2_CTRL_WHICH_REQUEST_VAL : 0,
+    };
+
+    if (!control || !count)
+        return 0;
+
+    if (ioctl(ctx->video_fd, VIDIOC_S_EXT_CTRLS, &controls) < 0)
+        return AVERROR(errno);
+
+    return 0;
+}
+
+int ff_v4l2_request_set_controls(AVCodecContext *avctx,
+                                 struct v4l2_ext_control *control, int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    int ret;
+
+    ret = ff_v4l2_request_set_request_controls(ctx, -1, control, count);
+    if (ret < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s): %s (%d)\n",
+               count, strerror(errno), errno);
+    }
+
+    return ret;
+}
+
+static int v4l2_request_buffer_alloc(V4L2RequestContext *ctx,
+                                     V4L2RequestBuffer *buf,
+                                     enum v4l2_buf_type type)
+{
+    struct v4l2_plane planes[1] = {};
+    struct v4l2_create_buffers buffers = {
+        .count = 1,
+        .memory = V4L2_MEMORY_MMAP,
+        .format.type = type,
+    };
+
+    // Get format details for the buffer to be created
+    if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &buffers.format) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to get format of type %d: %s (%d)\n",
+               type, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    // Create the buffer
+    if (ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to create buffer of type %d: %s (%d)\n",
+               type, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
+        buf->width = buffers.format.fmt.pix_mp.width;
+        buf->height = buffers.format.fmt.pix_mp.height;
+        buf->size = buffers.format.fmt.pix_mp.plane_fmt[0].sizeimage;
+        buf->buffer.length = 1;
+        buf->buffer.m.planes = planes;
+    } else {
+        buf->width = buffers.format.fmt.pix.width;
+        buf->height = buffers.format.fmt.pix.height;
+        buf->size = buffers.format.fmt.pix.sizeimage;
+    }
+
+    buf->index = buffers.index;
+    buf->capabilities = buffers.capabilities;
+    buf->used = 0;
+
+    buf->buffer.type = type;
+    buf->buffer.memory = V4L2_MEMORY_MMAP;
+    buf->buffer.index = buf->index;
+
+    // Query more details of the created buffer
+    if (ioctl(ctx->video_fd, VIDIOC_QUERYBUF, &buf->buffer) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to query buffer %d of type %d: %s (%d)\n",
+               buf->index, type, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    // Output buffers is mapped and capture buffers is exported
+    if (V4L2_TYPE_IS_OUTPUT(type)) {
+        off_t offset = V4L2_TYPE_IS_MULTIPLANAR(type) ?
+                       buf->buffer.m.planes[0].m.mem_offset :
+                       buf->buffer.m.offset;
+        void *addr = mmap(NULL, buf->size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                          ctx->video_fd, offset);
+        if (addr == MAP_FAILED) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to map output buffer %d: %s (%d)\n",
+                   buf->index, strerror(errno), errno);
+            return AVERROR(errno);
+        }
+
+        // Raw bitstream data is appended to output buffers
+        buf->addr = (uint8_t *)addr;
+    } else {
+        struct v4l2_exportbuffer exportbuffer = {
+            .type = type,
+            .index = buf->index,
+            .flags = O_RDONLY,
+        };
+
+        if (ioctl(ctx->video_fd, VIDIOC_EXPBUF, &exportbuffer) < 0) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to export capture buffer %d: %s (%d)\n",
+                   buf->index, strerror(errno), errno);
+            return AVERROR(errno);
+        }
+
+        // Used in the AVDRMFrameDescriptor for decoded frames
+        buf->fd = exportbuffer.fd;
+
+        /*
+         * Use buffer index as base for V4L2 frame reference.
+         * This works because a capture buffer is closely tied to a AVFrame
+         * and FFmpeg handle all frame reference tracking for us.
+         */
+        buf->buffer.timestamp.tv_usec = buf->index + 1;
+    }
+
+    return 0;
+}
+
+static void v4l2_request_buffer_free(V4L2RequestBuffer *buf)
+{
+    // Unmap output buffers
+    if (buf->addr) {
+        munmap(buf->addr, buf->size);
+        buf->addr = NULL;
+    }
+
+    // Close exported capture buffers or requests for output buffers
+    if (buf->fd >= 0) {
+        close(buf->fd);
+        buf->fd = -1;
+    }
+}
+
+static void v4l2_request_frame_free(void *opaque, uint8_t *data)
+{
+    V4L2RequestFrameDescriptor *desc = (V4L2RequestFrameDescriptor *)data;
+
+    v4l2_request_buffer_free(&desc->capture);
+
+    av_free(data);
+}
+
+static AVBufferRef *v4l2_request_frame_alloc(void *opaque, size_t size)
+{
+    V4L2RequestContext *ctx = opaque;
+    V4L2RequestFrameDescriptor *desc;
+    AVBufferRef *ref;
+    uint8_t *data;
+
+    data = av_mallocz(size);
+    if (!data)
+        return NULL;
+
+    ref = av_buffer_create(data, size, v4l2_request_frame_free, ctx, 0);
+    if (!ref) {
+        av_free(data);
+        return NULL;
+    }
+
+    desc = (V4L2RequestFrameDescriptor *)data;
+    desc->capture.fd = -1;
+
+    // Create a V4L2 capture buffer for this AVFrame
+    if (v4l2_request_buffer_alloc(ctx, &desc->capture, ctx->format.type) < 0) {
+        av_buffer_unref(&ref);
+        return NULL;
+    }
+
+    // TODO: Set AVDRMFrameDescriptor of this AVFrame
+
+    return ref;
+}
+
+static void v4l2_request_hwframe_ctx_free(AVHWFramesContext *hwfc)
+{
+    av_buffer_pool_uninit(&hwfc->pool);
+}
+
+int ff_v4l2_request_frame_params(AVCodecContext *avctx,
+                                 AVBufferRef *hw_frames_ctx)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    AVHWFramesContext *hwfc = (AVHWFramesContext *)hw_frames_ctx->data;
+
+    hwfc->format = AV_PIX_FMT_DRM_PRIME;
+    // TODO: Set sw_format based on capture buffer format
+    hwfc->sw_format = AV_PIX_FMT_NONE;
+
+    if (V4L2_TYPE_IS_MULTIPLANAR(ctx->format.type)) {
+        hwfc->width = ctx->format.fmt.pix_mp.width;
+        hwfc->height = ctx->format.fmt.pix_mp.height;
+    } else {
+        hwfc->width = ctx->format.fmt.pix.width;
+        hwfc->height = ctx->format.fmt.pix.height;
+    }
+
+    hwfc->pool = av_buffer_pool_init2(sizeof(V4L2RequestFrameDescriptor), ctx,
+                                      v4l2_request_frame_alloc, NULL);
+    if (!hwfc->pool)
+        return AVERROR(ENOMEM);
+
+    hwfc->free = v4l2_request_hwframe_ctx_free;
+
+    hwfc->initial_pool_size = 1;
+
+    switch (avctx->codec_id) {
+    case AV_CODEC_ID_VP9:
+        hwfc->initial_pool_size += 8;
+        break;
+    case AV_CODEC_ID_VP8:
+        hwfc->initial_pool_size += 3;
+        break;
+    default:
+        hwfc->initial_pool_size += 2;
+    }
+
+    return 0;
+}
+
+int ff_v4l2_request_uninit(AVCodecContext *avctx)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+
+    if (ctx->video_fd >= 0) {
+        // TODO: Flush and wait on all pending requests
+
+        // Stop output queue
+        if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) {
+            av_log(ctx, AV_LOG_WARNING, "Failed to stop output streaming: %s (%d)\n",
+                   strerror(errno), errno);
+        }
+
+        // Stop capture queue
+        if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->format.type) < 0) {
+            av_log(ctx, AV_LOG_WARNING, "Failed to stop capture streaming: %s (%d)\n",
+                   strerror(errno), errno);
+        }
+
+        // Release output buffers
+        for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) {
+            v4l2_request_buffer_free(&ctx->output[i]);
+        }
+
+        // Close video device file descriptor
+        close(ctx->video_fd);
+        ctx->video_fd = -1;
+    }
+
+    // Ownership of media device file descriptor may belong to hwdevice
+    if (ctx->device_ref) {
+        av_buffer_unref(&ctx->device_ref);
+        ctx->media_fd = -1;
+    } else if (ctx->media_fd >= 0) {
+        close(ctx->media_fd);
+        ctx->media_fd = -1;
+    }
+
+    ff_mutex_destroy(&ctx->mutex);
+
+    return 0;
+}
+
+static int v4l2_request_init_context(AVCodecContext *avctx)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    int ret;
+
+    // Initialize context state
+    ff_mutex_init(&ctx->mutex, NULL);
+    for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) {
+        ctx->output[i].index = i;
+        ctx->output[i].fd = -1;
+    }
+    atomic_init(&ctx->next_output, 0);
+    atomic_init(&ctx->queued_output, 0);
+    atomic_init(&ctx->queued_request, 0);
+    atomic_init(&ctx->queued_capture, 0);
+
+    // Get format details for capture buffers
+    if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &ctx->format) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to get capture format: %s (%d)\n",
+               strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    // Create frame context and allocate initial capture buffers
+    ret = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_V4L2REQUEST);
+    if (ret < 0)
+        goto fail;
+
+    // Allocate output buffers for circular queue
+    for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) {
+        ret = v4l2_request_buffer_alloc(ctx, &ctx->output[i], ctx->output_type);
+        if (ret < 0)
+            goto fail;
+    }
+
+    // Allocate requests for circular queue
+    for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) {
+        if (ioctl(ctx->media_fd, MEDIA_IOC_REQUEST_ALLOC, &ctx->output[i].fd) < 0) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to allocate request %d: %s (%d)\n",
+                   i, strerror(errno), errno);
+            ret = AVERROR(errno);
+            goto fail;
+        }
+    }
+
+    // Start output queue
+    if (ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->output_type) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to start output streaming: %s (%d)\n",
+               strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    // Start capture queue
+    if (ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->format.type) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to start capture streaming: %s (%d)\n",
+               strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    return 0;
+
+fail:
+    ff_v4l2_request_uninit(avctx);
+    return ret;
+}
+
+int ff_v4l2_request_init(AVCodecContext *avctx,
+                         uint32_t pixelformat, uint32_t buffersize,
+                         struct v4l2_ext_control *control, int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    AVV4L2RequestDeviceContext *hwctx = NULL;
+
+    // Set initial default values
+    ctx->av_class = &v4l2_request_context_class;
+    ctx->media_fd = -1;
+    ctx->video_fd = -1;
+
+    // Try to use media device file descriptor opened and owned by hwdevice
+    if (avctx->hw_device_ctx) {
+        AVHWDeviceContext *device_ctx = (AVHWDeviceContext *)avctx->hw_device_ctx->data;
+        if (device_ctx->type == AV_HWDEVICE_TYPE_V4L2REQUEST && device_ctx->hwctx) {
+            hwctx = device_ctx->hwctx;
+            ctx->media_fd = hwctx->media_fd;
+        }
+    }
+
+    // TODO: Probe for a capable media and video device
+
+    // Transfer (or return) ownership of media device file descriptor to hwdevice
+    if (hwctx) {
+        hwctx->media_fd = ctx->media_fd;
+        ctx->device_ref = av_buffer_ref(avctx->hw_device_ctx);
+    }
+
+    // Create buffers and finalize init
+    return v4l2_request_init_context(avctx);
+}
diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h
new file mode 100644
index 0000000000..621caaf28c
--- /dev/null
+++ b/libavcodec/v4l2_request.h
@@ -0,0 +1,84 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_V4L2_REQUEST_H
+#define AVCODEC_V4L2_REQUEST_H
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+#include <linux/videodev2.h>
+
+#include "libavutil/hwcontext_drm.h"
+#include "libavutil/thread.h"
+
+typedef struct V4L2RequestBuffer {
+    int index;
+    int fd;
+    uint8_t *addr;
+    uint32_t width;
+    uint32_t height;
+    uint32_t size;
+    uint32_t used;
+    uint32_t capabilities;
+    struct v4l2_buffer buffer;
+} V4L2RequestBuffer;
+
+typedef struct V4L2RequestContext {
+    const AVClass *av_class;
+    AVBufferRef *device_ref;
+    int media_fd;
+    int video_fd;
+    struct v4l2_format format;
+    enum v4l2_buf_type output_type;
+    AVMutex mutex;
+    V4L2RequestBuffer output[4];
+    atomic_int_least8_t next_output;
+    atomic_uint_least32_t queued_output;
+    atomic_uint_least32_t queued_request;
+    atomic_uint_least64_t queued_capture;
+    int (*post_probe)(AVCodecContext *avctx);
+} V4L2RequestContext;
+
+typedef struct V4L2RequestPictureContext {
+    V4L2RequestBuffer *output;
+    V4L2RequestBuffer *capture;
+} V4L2RequestPictureContext;
+
+int ff_v4l2_request_query_control(AVCodecContext *avctx,
+                                  struct v4l2_query_ext_ctrl *control);
+
+int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx,
+                                                uint32_t id);
+
+int ff_v4l2_request_set_request_controls(V4L2RequestContext *ctx, int request_fd,
+                                         struct v4l2_ext_control *control, int count);
+
+int ff_v4l2_request_set_controls(AVCodecContext *avctx,
+                                 struct v4l2_ext_control *control, int count);
+
+int ff_v4l2_request_frame_params(AVCodecContext *avctx,
+                                 AVBufferRef *hw_frames_ctx);
+
+int ff_v4l2_request_uninit(AVCodecContext *avctx);
+
+int ff_v4l2_request_init(AVCodecContext *avctx,
+                         uint32_t pixelformat, uint32_t buffersize,
+                         struct v4l2_ext_control *control, int count);
+
+#endif /* AVCODEC_V4L2_REQUEST_H */
diff --git a/libavcodec/v4l2_request_internal.h b/libavcodec/v4l2_request_internal.h
new file mode 100644
index 0000000000..554b663ab4
--- /dev/null
+++ b/libavcodec/v4l2_request_internal.h
@@ -0,0 +1,42 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_V4L2_REQUEST_INTERNAL_H
+#define AVCODEC_V4L2_REQUEST_INTERNAL_H
+
+#include <linux/media.h>
+
+#include "internal.h"
+#include "v4l2_request.h"
+
+typedef struct V4L2RequestFrameDescriptor {
+    AVDRMFrameDescriptor base;
+    V4L2RequestBuffer capture;
+} V4L2RequestFrameDescriptor;
+
+static inline V4L2RequestContext *v4l2_request_context(AVCodecContext *avctx)
+{
+    return (V4L2RequestContext *)avctx->internal->hwaccel_priv_data;
+}
+
+static inline V4L2RequestFrameDescriptor *v4l2_request_framedesc(AVFrame *frame)
+{
+    return (V4L2RequestFrameDescriptor *)frame->data[0];
+}
+
+#endif /* AVCODEC_V4L2_REQUEST_INTERNAL_H */

From f2779c95d7416be33aa3b50f1c7f1f33f5b9aad5 Mon Sep 17 00:00:00 2001
From: Jonas Karlman <jonas@kwiboo.se>
Date: Tue, 6 Aug 2024 09:06:02 +0000
Subject: [PATCH 03/11] avcodec/v4l2request: Probe for a capable media and
 video device

Probe all media devices and its linked video devices to locate a video
device that support stateless decoding of the specific codec using the
V4L2 Request API.

When AVV4L2RequestDeviceContext.media_fd is a valid file descriptor only
video devices for the opened media device is probed.
E.g. using a -init_hw_device v4l2request:/dev/media1 parameter.

A video device is deemed capable when all tests pass, e.g. kernel driver
support the codec, FFmpeg v4l2-request code know about the buffer format
and kernel driver report that the frame size is supported.

A codec specific hook, post_probe, is supported to e.g. test for codec
specific limitations, PROFILE and LEVEL controls.

Basic flow for initialization follow the kernel Memory-to-memory
Stateless Video Decoder Interface > Initialization [1].

[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization

Co-developed-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                          |   4 +-
 libavcodec/Makefile                |   2 +-
 libavcodec/v4l2_request.c          |  18 +-
 libavcodec/v4l2_request_internal.h |   9 +
 libavcodec/v4l2_request_probe.c    | 614 +++++++++++++++++++++++++++++
 5 files changed, 641 insertions(+), 6 deletions(-)
 create mode 100644 libavcodec/v4l2_request_probe.c

diff --git a/configure b/configure
index b8f82f0fbf..98a342d109 100755
--- a/configure
+++ b/configure
@@ -1966,6 +1966,7 @@ EXTERNAL_LIBRARY_LIST="
     libtorch
     libtwolame
     libuavs3d
+    libudev
     libv4l2
     libvmaf
     libvorbis
@@ -3148,7 +3149,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
 ffnvcodec_deps_any="libdl LoadLibrary"
 mediacodec_deps="android mediandk"
 nvdec_deps="ffnvcodec"
-v4l2_request_deps="linux_media_h v4l2_timeval_to_ns"
+v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev"
 vaapi_x11_deps="xlib_x11"
 videotoolbox_hwaccel_deps="videotoolbox pthreads"
 videotoolbox_hwaccel_extralibs="-framework QuartzCore"
@@ -7202,6 +7203,7 @@ fi
 
 if enabled v4l2_request; then
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
+    check_pkg_config libudev libudev libudev.h udev_new
 fi
 
 check_headers sys/videoio.h
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index af60923827..ebd2dc54b9 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -176,7 +176,7 @@ OBJS-$(CONFIG_VP3DSP)                  += vp3dsp.o
 OBJS-$(CONFIG_VP56DSP)                 += vp56dsp.o
 OBJS-$(CONFIG_VP8DSP)                  += vp8dsp.o
 OBJS-$(CONFIG_V4L2_M2M)                += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
-OBJS-$(CONFIG_V4L2_REQUEST)            += v4l2_request.o
+OBJS-$(CONFIG_V4L2_REQUEST)            += v4l2_request.o v4l2_request_probe.o
 OBJS-$(CONFIG_WMA_FREQS)               += wma_freqs.o
 OBJS-$(CONFIG_WMV2DSP)                 += wmv2dsp.o
 
diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c
index 436c8de4a4..6f44c5d826 100644
--- a/libavcodec/v4l2_request.c
+++ b/libavcodec/v4l2_request.c
@@ -244,7 +244,11 @@ static AVBufferRef *v4l2_request_frame_alloc(void *opaque, size_t size)
         return NULL;
     }
 
-    // TODO: Set AVDRMFrameDescriptor of this AVFrame
+    // Set AVDRMFrameDescriptor of this AVFrame
+    if (ff_v4l2_request_set_drm_descriptor(desc, &ctx->format) < 0) {
+        av_buffer_unref(&ref);
+        return NULL;
+    }
 
     return ref;
 }
@@ -261,8 +265,7 @@ int ff_v4l2_request_frame_params(AVCodecContext *avctx,
     AVHWFramesContext *hwfc = (AVHWFramesContext *)hw_frames_ctx->data;
 
     hwfc->format = AV_PIX_FMT_DRM_PRIME;
-    // TODO: Set sw_format based on capture buffer format
-    hwfc->sw_format = AV_PIX_FMT_NONE;
+    hwfc->sw_format = ff_v4l2_request_get_sw_format(&ctx->format);
 
     if (V4L2_TYPE_IS_MULTIPLANAR(ctx->format.type)) {
         hwfc->width = ctx->format.fmt.pix_mp.width;
@@ -413,6 +416,7 @@ int ff_v4l2_request_init(AVCodecContext *avctx,
 {
     V4L2RequestContext *ctx = v4l2_request_context(avctx);
     AVV4L2RequestDeviceContext *hwctx = NULL;
+    int ret;
 
     // Set initial default values
     ctx->av_class = &v4l2_request_context_class;
@@ -428,7 +432,13 @@ int ff_v4l2_request_init(AVCodecContext *avctx,
         }
     }
 
-    // TODO: Probe for a capable media and video device
+    // Probe for a capable media and video device for the V4L2 codec pixelformat
+    ret = ff_v4l2_request_probe(avctx, pixelformat, buffersize, control, count);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_INFO, "No V4L2 video device found for %s\n",
+               av_fourcc2str(pixelformat));
+        return ret;
+    }
 
     // Transfer (or return) ownership of media device file descriptor to hwdevice
     if (hwctx) {
diff --git a/libavcodec/v4l2_request_internal.h b/libavcodec/v4l2_request_internal.h
index 554b663ab4..a5c3de22ef 100644
--- a/libavcodec/v4l2_request_internal.h
+++ b/libavcodec/v4l2_request_internal.h
@@ -39,4 +39,13 @@ static inline V4L2RequestFrameDescriptor *v4l2_request_framedesc(AVFrame *frame)
     return (V4L2RequestFrameDescriptor *)frame->data[0];
 }
 
+enum AVPixelFormat ff_v4l2_request_get_sw_format(struct v4l2_format *format);
+
+int ff_v4l2_request_set_drm_descriptor(V4L2RequestFrameDescriptor *framedesc,
+                                       struct v4l2_format *format);
+
+int ff_v4l2_request_probe(AVCodecContext *avctx, uint32_t pixelformat,
+                          uint32_t buffersize, struct v4l2_ext_control *control,
+                          int count);
+
 #endif /* AVCODEC_V4L2_REQUEST_INTERNAL_H */
diff --git a/libavcodec/v4l2_request_probe.c b/libavcodec/v4l2_request_probe.c
new file mode 100644
index 0000000000..941042ec04
--- /dev/null
+++ b/libavcodec/v4l2_request_probe.c
@@ -0,0 +1,614 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <drm_fourcc.h>
+#include <libudev.h>
+
+#include "libavutil/hwcontext_v4l2request.h"
+#include "libavutil/mem.h"
+#include "v4l2_request_internal.h"
+
+static const struct {
+    uint32_t pixelformat;
+    enum AVPixelFormat sw_format;
+    uint32_t drm_format;
+    uint64_t format_modifier;
+} v4l2_request_capture_pixelformats[] = {
+    { V4L2_PIX_FMT_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_LINEAR },
+#if defined(V4L2_PIX_FMT_NV12_32L32)
+    { V4L2_PIX_FMT_NV12_32L32, AV_PIX_FMT_NONE, DRM_FORMAT_NV12, DRM_FORMAT_MOD_ALLWINNER_TILED },
+#endif
+#if defined(V4L2_PIX_FMT_NV15) && defined(DRM_FORMAT_NV15)
+    { V4L2_PIX_FMT_NV15, AV_PIX_FMT_NONE, DRM_FORMAT_NV15, DRM_FORMAT_MOD_LINEAR },
+#endif
+    { V4L2_PIX_FMT_NV16, AV_PIX_FMT_NV16, DRM_FORMAT_NV16, DRM_FORMAT_MOD_LINEAR },
+#if defined(V4L2_PIX_FMT_NV20) && defined(DRM_FORMAT_NV20)
+    { V4L2_PIX_FMT_NV20, AV_PIX_FMT_NONE, DRM_FORMAT_NV20, DRM_FORMAT_MOD_LINEAR },
+#endif
+#if defined(V4L2_PIX_FMT_P010) && defined(DRM_FORMAT_P010)
+    { V4L2_PIX_FMT_P010, AV_PIX_FMT_P010, DRM_FORMAT_P010, DRM_FORMAT_MOD_LINEAR },
+#endif
+#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128)
+    {
+        .pixelformat = V4L2_PIX_FMT_NV12_COL128,
+        .sw_format = AV_PIX_FMT_NONE,
+        .drm_format = DRM_FORMAT_NV12,
+        .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128,
+    },
+#if defined(DRM_FORMAT_P030)
+    {
+        .pixelformat = V4L2_PIX_FMT_NV12_10_COL128,
+        .sw_format = AV_PIX_FMT_NONE,
+        .drm_format = DRM_FORMAT_P030,
+        .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128,
+    },
+#endif
+#endif
+#if defined(V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT)
+    {
+        .pixelformat = V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT,
+        .sw_format = AV_PIX_FMT_NONE,
+        .drm_format = DRM_FORMAT_YUV420_10BIT,
+        .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |
+                                                   AFBC_FORMAT_MOD_SPARSE |
+                                                   AFBC_FORMAT_MOD_SPLIT),
+    },
+#endif
+#if defined(V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT)
+    {
+        .pixelformat = V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT,
+        .sw_format = AV_PIX_FMT_NONE,
+        .drm_format = DRM_FORMAT_YUV420_8BIT,
+        .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |
+                                                   AFBC_FORMAT_MOD_SPARSE |
+                                                   AFBC_FORMAT_MOD_SPLIT),
+    },
+#endif
+};
+
+enum AVPixelFormat ff_v4l2_request_get_sw_format(struct v4l2_format *format)
+{
+    uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                           format->fmt.pix_mp.pixelformat :
+                           format->fmt.pix.pixelformat;
+
+    for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) {
+        if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat)
+            return v4l2_request_capture_pixelformats[i].sw_format;
+    }
+
+    return AV_PIX_FMT_NONE;
+}
+
+int ff_v4l2_request_set_drm_descriptor(V4L2RequestFrameDescriptor *framedesc,
+                                       struct v4l2_format *format)
+{
+    AVDRMFrameDescriptor *desc = &framedesc->base;
+    AVDRMLayerDescriptor *layer = &desc->layers[0];
+    uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                           format->fmt.pix_mp.pixelformat :
+                           format->fmt.pix.pixelformat;
+
+    // Set drm format and format modifier
+    layer->format = 0;
+    for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) {
+        if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) {
+            layer->format = v4l2_request_capture_pixelformats[i].drm_format;
+            desc->objects[0].format_modifier =
+                        v4l2_request_capture_pixelformats[i].format_modifier;
+            break;
+        }
+    }
+
+    if (!layer->format)
+        return AVERROR(ENOENT);
+
+    desc->nb_objects = 1;
+    desc->objects[0].fd = framedesc->capture.fd;
+    desc->objects[0].size = framedesc->capture.size;
+
+    desc->nb_layers = 1;
+    layer->nb_planes = 1;
+
+    layer->planes[0].object_index = 0;
+    layer->planes[0].offset = 0;
+    layer->planes[0].pitch = V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                             format->fmt.pix_mp.plane_fmt[0].bytesperline :
+                             format->fmt.pix.bytesperline;
+
+    // AFBC formats only use 1 plane, remaining use 2 planes
+    if ((desc->objects[0].format_modifier >> 56) != DRM_FORMAT_MOD_VENDOR_ARM) {
+        layer->nb_planes = 2;
+        layer->planes[1].object_index = 0;
+        layer->planes[1].offset = layer->planes[0].pitch *
+                                  (V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                                   format->fmt.pix_mp.height :
+                                   format->fmt.pix.height);
+        layer->planes[1].pitch = layer->planes[0].pitch;
+    }
+
+#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128)
+    // Raspberry Pi formats need special handling
+    if (pixelformat == V4L2_PIX_FMT_NV12_COL128 ||
+        pixelformat == V4L2_PIX_FMT_NV12_10_COL128) {
+        desc->objects[0].format_modifier =
+            DRM_FORMAT_MOD_BROADCOM_SAND128_COL_HEIGHT(layer->planes[0].pitch);
+        layer->planes[1].offset = 128 *
+                                  (V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                                   format->fmt.pix_mp.height :
+                                   format->fmt.pix.height);
+        layer->planes[0].pitch = (V4L2_TYPE_IS_MULTIPLANAR(format->type) ?
+                                  format->fmt.pix_mp.width :
+                                  format->fmt.pix.width);
+        if (pixelformat == V4L2_PIX_FMT_NV12_10_COL128)
+            layer->planes[0].pitch *= 2;
+        layer->planes[1].pitch = layer->planes[0].pitch;
+    }
+#endif
+
+    return 0;
+}
+
+static int v4l2_request_set_format(AVCodecContext *avctx,
+                                   enum v4l2_buf_type type,
+                                   uint32_t pixelformat,
+                                   uint32_t buffersize)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct v4l2_format format = {
+        .type = type,
+    };
+
+    if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
+        format.fmt.pix_mp.width = avctx->coded_width;
+        format.fmt.pix_mp.height = avctx->coded_height;
+        format.fmt.pix_mp.pixelformat = pixelformat;
+        format.fmt.pix_mp.plane_fmt[0].sizeimage = buffersize;
+        format.fmt.pix_mp.num_planes = 1;
+    } else {
+        format.fmt.pix.width = avctx->coded_width;
+        format.fmt.pix.height = avctx->coded_height;
+        format.fmt.pix.pixelformat = pixelformat;
+        format.fmt.pix.sizeimage = buffersize;
+    }
+
+    if (ioctl(ctx->video_fd, VIDIOC_S_FMT, &format) < 0)
+        return AVERROR(errno);
+
+    return 0;
+}
+
+static int v4l2_request_select_capture_format(AVCodecContext *avctx)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    enum v4l2_buf_type type = ctx->format.type;
+    struct v4l2_format format = {
+        .type = type,
+    };
+    struct v4l2_fmtdesc fmtdesc = {
+        .index = 0,
+        .type = type,
+    };
+    uint32_t pixelformat;
+
+    // Get the driver preferred (or default) format
+    if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &format) < 0)
+        return AVERROR(errno);
+
+    pixelformat = V4L2_TYPE_IS_MULTIPLANAR(type) ?
+                  format.fmt.pix_mp.pixelformat :
+                  format.fmt.pix.pixelformat;
+
+    // Use the driver preferred format when it is supported
+    for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) {
+        if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat)
+            return v4l2_request_set_format(avctx, type, pixelformat, 0);
+    }
+
+    // Otherwise, use first format that is supported
+    while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) {
+        for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) {
+            if (fmtdesc.pixelformat == v4l2_request_capture_pixelformats[i].pixelformat)
+                return v4l2_request_set_format(avctx, type, fmtdesc.pixelformat, 0);
+        }
+
+        fmtdesc.index++;
+    }
+
+    return AVERROR(errno);
+}
+
+static int v4l2_request_try_framesize(AVCodecContext *avctx,
+                                      uint32_t pixelformat)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct v4l2_frmsizeenum frmsize = {
+        .index = 0,
+        .pixel_format = pixelformat,
+    };
+
+    // Enumerate and check if frame size is supported
+    while (ioctl(ctx->video_fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) >= 0) {
+        if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE &&
+            frmsize.discrete.width == avctx->coded_width &&
+            frmsize.discrete.height == avctx->coded_height) {
+            return 0;
+        } else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE ||
+                    frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) &&
+                   avctx->coded_width <= frmsize.stepwise.max_width &&
+                   avctx->coded_height <= frmsize.stepwise.max_height &&
+                   avctx->coded_width % frmsize.stepwise.step_width == 0 &&
+                   avctx->coded_height % frmsize.stepwise.step_height == 0) {
+            return 0;
+        }
+
+        frmsize.index++;
+    }
+
+    return AVERROR(errno);
+}
+
+static int v4l2_request_try_format(AVCodecContext *avctx,
+                                   enum v4l2_buf_type type,
+                                   uint32_t pixelformat)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct v4l2_fmtdesc fmtdesc = {
+        .index = 0,
+        .type = type,
+    };
+
+    // Enumerate and check if format is supported
+    while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) {
+        if (fmtdesc.pixelformat == pixelformat)
+            return 0;
+
+        fmtdesc.index++;
+    }
+
+    return AVERROR(errno);
+}
+
+static int v4l2_request_probe_video_device(const char *path,
+                                           AVCodecContext *avctx,
+                                           uint32_t pixelformat,
+                                           uint32_t buffersize,
+                                           struct v4l2_ext_control *control,
+                                           int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct v4l2_capability capability;
+    struct v4l2_create_buffers buffers = {
+        .count = 0,
+        .memory = V4L2_MEMORY_MMAP,
+    };
+    unsigned int capabilities;
+    int ret;
+
+    /*
+     * Open video device in non-blocking mode to support decoding using
+     * multiple queued requests, required for e.g. multi stage decoding.
+     */
+    ctx->video_fd = open(path, O_RDWR | O_NONBLOCK);
+    if (ctx->video_fd < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to open video device %s: %s (%d)\n",
+               path, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    // Query capabilities of the video device
+    if (ioctl(ctx->video_fd, VIDIOC_QUERYCAP, &capability) < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to query capabilities of %s: %s (%d)\n",
+               path, strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    // Use device capabilities when needed
+    if (capability.capabilities & V4L2_CAP_DEVICE_CAPS)
+        capabilities = capability.device_caps;
+    else
+        capabilities = capability.capabilities;
+
+    // Ensure streaming is supported on the video device
+    if ((capabilities & V4L2_CAP_STREAMING) != V4L2_CAP_STREAMING) {
+        av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing streaming capability\n", path);
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    // Ensure multi- or single-planar API can be used
+    if ((capabilities & V4L2_CAP_VIDEO_M2M_MPLANE) == V4L2_CAP_VIDEO_M2M_MPLANE) {
+        ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+        ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    } else if ((capabilities & V4L2_CAP_VIDEO_M2M) == V4L2_CAP_VIDEO_M2M) {
+        ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+        ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+    } else {
+        av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing mem2mem capability\n", path);
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    // Query output buffer capabilities
+    buffers.format.type = ctx->output_type;
+    if (ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Failed to query output buffer capabilities of %s: %s (%d)\n",
+               path, strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    // Ensure requests can be used
+    if ((buffers.capabilities & V4L2_BUF_CAP_SUPPORTS_REQUESTS) !=
+        V4L2_BUF_CAP_SUPPORTS_REQUESTS) {
+        av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing support for requests\n", path);
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    // Ensure the codec pixelformat can be used
+    ret = v4l2_request_try_format(avctx, ctx->output_type, pixelformat);
+    if (ret < 0) {
+        av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing support for pixelformat %s\n",
+               path, av_fourcc2str(pixelformat));
+        goto fail;
+    }
+
+    // Ensure frame size is supported, when driver support ENUM_FRAMESIZES
+    ret = v4l2_request_try_framesize(avctx, pixelformat);
+    if (ret < 0 && ret != AVERROR(ENOTTY)) {
+        av_log(ctx, AV_LOG_VERBOSE,
+               "Device %s is missing support for frame size %dx%d of pixelformat %s\n",
+               path, avctx->coded_width, avctx->coded_height, av_fourcc2str(pixelformat));
+        goto fail;
+    }
+
+    // Set the codec pixelformat and output buffersize to be used
+    ret = v4l2_request_set_format(avctx, ctx->output_type, pixelformat, buffersize);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Failed to set output pixelformat %s of %s: %s (%d)\n",
+               av_fourcc2str(pixelformat), path, strerror(errno), errno);
+        goto fail;
+    }
+
+    /*
+     * Set any codec specific controls that can help assist the driver
+     * make a decision on what capture buffer format can be used.
+     */
+    ret = ff_v4l2_request_set_controls(avctx, control, count);
+    if (ret < 0)
+        goto fail;
+
+    // Select a capture buffer format known to the hwaccel
+    ret = v4l2_request_select_capture_format(avctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_VERBOSE,
+               "Failed to select a capture format for %s of %s: %s (%d)\n",
+               av_fourcc2str(pixelformat), path, strerror(errno), errno);
+        goto fail;
+    }
+
+    // Check codec specific controls, e.g. profile and level
+    if (ctx->post_probe) {
+        ret = ctx->post_probe(avctx);
+        if (ret < 0)
+            goto fail;
+    }
+
+    // All tests passed, video device should be capable
+    return 0;
+
+fail:
+    if (ctx->video_fd >= 0) {
+        close(ctx->video_fd);
+        ctx->video_fd = -1;
+    }
+    return ret;
+}
+
+static int v4l2_request_probe_video_devices(struct udev *udev,
+                                            AVCodecContext *avctx,
+                                            uint32_t pixelformat,
+                                            uint32_t buffersize,
+                                            struct v4l2_ext_control *control,
+                                            int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct media_device_info device_info;
+    struct media_v2_topology topology = {0};
+    struct media_v2_interface *interfaces;
+    struct udev_device *device;
+    const char *path;
+    dev_t devnum;
+    int ret;
+
+    if (ioctl(ctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0)
+        return AVERROR(errno);
+
+    if (ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n",
+               strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    if (!topology.num_interfaces)
+        return AVERROR(ENOENT);
+
+    interfaces = av_calloc(topology.num_interfaces, sizeof(struct media_v2_interface));
+    if (!interfaces)
+        return AVERROR(ENOMEM);
+
+    topology.ptr_interfaces = (__u64)(uintptr_t)interfaces;
+    if (ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n",
+               strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    ret = AVERROR(ENOENT);
+    for (int i = 0; i < topology.num_interfaces; i++) {
+        if (interfaces[i].intf_type != MEDIA_INTF_T_V4L_VIDEO)
+            continue;
+
+        devnum = makedev(interfaces[i].devnode.major, interfaces[i].devnode.minor);
+        device = udev_device_new_from_devnum(udev, 'c', devnum);
+        if (!device)
+            continue;
+
+        path = udev_device_get_devnode(device);
+        if (path)
+            ret = v4l2_request_probe_video_device(path, avctx, pixelformat,
+                                                  buffersize, control, count);
+        udev_device_unref(device);
+
+        // Stop when we have found a capable video device
+        if (!ret) {
+            av_log(avctx, AV_LOG_INFO,
+                   "Using V4L2 media driver %s (%u.%u.%u) for %s\n",
+                   device_info.driver,
+                   device_info.driver_version >> 16,
+                   (device_info.driver_version >> 8) & 0xff,
+                   device_info.driver_version & 0xff,
+                   av_fourcc2str(pixelformat));
+            break;
+        }
+    }
+
+fail:
+    av_free(interfaces);
+    return ret;
+}
+
+static int v4l2_request_probe_media_device(struct udev_device *device,
+                                           AVCodecContext *avctx,
+                                           uint32_t pixelformat,
+                                           uint32_t buffersize,
+                                           struct v4l2_ext_control *control,
+                                           int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    const char *path;
+    int ret;
+
+    path = udev_device_get_devnode(device);
+    if (!path)
+        return AVERROR(ENODEV);
+
+    // Open enumerated media device
+    ctx->media_fd = open(path, O_RDWR);
+    if (ctx->media_fd < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to open media device %s: %s (%d)\n",
+               path, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    // Probe video devices of current media device
+    ret = v4l2_request_probe_video_devices(udev_device_get_udev(device),
+                                           avctx, pixelformat,
+                                           buffersize, control, count);
+
+    // Cleanup when no capable video device was found
+    if (ret < 0) {
+        close(ctx->media_fd);
+        ctx->media_fd = -1;
+    }
+
+    return ret;
+}
+
+static int v4l2_request_probe_media_devices(struct udev *udev,
+                                            AVCodecContext *avctx,
+                                            uint32_t pixelformat,
+                                            uint32_t buffersize,
+                                            struct v4l2_ext_control *control,
+                                            int count)
+{
+    struct udev_enumerate *enumerate;
+    struct udev_list_entry *devices;
+    struct udev_list_entry *entry;
+    struct udev_device *device;
+    int ret;
+
+    enumerate = udev_enumerate_new(udev);
+    if (!enumerate)
+        return AVERROR(ENOMEM);
+
+    udev_enumerate_add_match_subsystem(enumerate, "media");
+    udev_enumerate_scan_devices(enumerate);
+    devices = udev_enumerate_get_list_entry(enumerate);
+
+    ret = AVERROR(ENOENT);
+    udev_list_entry_foreach(entry, devices) {
+        const char *path = udev_list_entry_get_name(entry);
+        if (!path)
+            continue;
+
+        device = udev_device_new_from_syspath(udev, path);
+        if (!device)
+            continue;
+
+        // Probe media device for a capable video device
+        ret = v4l2_request_probe_media_device(device, avctx, pixelformat,
+                                              buffersize, control, count);
+        udev_device_unref(device);
+
+        // Stop when we have found a capable media and video device
+        if (!ret)
+            break;
+    }
+
+    udev_enumerate_unref(enumerate);
+    return ret;
+}
+
+int ff_v4l2_request_probe(AVCodecContext *avctx,
+                          uint32_t pixelformat, uint32_t buffersize,
+                          struct v4l2_ext_control *control, int count)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct udev *udev;
+    int ret;
+
+    udev = udev_new();
+    if (!udev)
+        return AVERROR(ENOMEM);
+
+    if (ctx->media_fd >= 0) {
+        // Probe video devices of current media device
+        ret = v4l2_request_probe_video_devices(udev, avctx, pixelformat,
+                                               buffersize, control, count);
+    } else {
+        // Probe all media devices (auto-detect)
+        ret = v4l2_request_probe_media_devices(udev, avctx, pixelformat,
+                                               buffersize, control, count);
+    }
+
+    udev_unref(udev);
+    return ret;
+}

From 917168b391fa76cff0b7b74050bd90fa60e07b8c Mon Sep 17 00:00:00 2001
From: Jonas Karlman <jonas@kwiboo.se>
Date: Tue, 6 Aug 2024 09:06:03 +0000
Subject: [PATCH 04/11] avcodec/v4l2request: Add common decode support for
 hwaccels

Add common support for decoding using the V4L2 Request API.

Basic flow for decoding follow the kernel Memory-to-memory Stateless
Video Decoder Interface > Decoding [1].

A codec hwaccel typically handle decoding as follow:

In start_frame next OUTPUT buffer and its related request object is
picked from a circular queue and any codec specific CONTROLs is prepared.

In decode_slice the slice bitstream data is appended to the OUTPUT
buffer.

In end_frame a CAPTURE buffer tied to the AVFrame is queued, it will be
used as the decoding target by the driver / hw decoder. The prepared
codec specific CONTROLs get queued as part of the request object.
Finally the request object is submitted for decoding.

For slice based hw decoders only the request for the final slice of the
frame is submitted in end_frame, remaining is submitted in decode_slice.

[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#decoding

Co-developed-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                        |   3 +-
 libavcodec/Makefile              |   2 +-
 libavcodec/hwconfig.h            |   2 +
 libavcodec/v4l2_request.c        |   3 +-
 libavcodec/v4l2_request.h        |  23 ++
 libavcodec/v4l2_request_decode.c | 459 +++++++++++++++++++++++++++++++
 6 files changed, 489 insertions(+), 3 deletions(-)
 create mode 100644 libavcodec/v4l2_request_decode.c

diff --git a/configure b/configure
index 98a342d109..b74da32e8b 100755
--- a/configure
+++ b/configure
@@ -3149,7 +3149,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
 ffnvcodec_deps_any="libdl LoadLibrary"
 mediacodec_deps="android mediandk"
 nvdec_deps="ffnvcodec"
-v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev"
+v4l2_request_deps="linux_media_h v4l2_timeval_to_ns v4l2_m2m_hold_capture_buf libdrm libudev"
 vaapi_x11_deps="xlib_x11"
 videotoolbox_hwaccel_deps="videotoolbox pthreads"
 videotoolbox_hwaccel_extralibs="-framework QuartzCore"
@@ -7202,6 +7202,7 @@ if enabled v4l2_m2m; then
 fi
 
 if enabled v4l2_request; then
+    check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
     check_pkg_config libudev libudev libudev.h udev_new
 fi
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index ebd2dc54b9..6f2229b1b6 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -176,7 +176,7 @@ OBJS-$(CONFIG_VP3DSP)                  += vp3dsp.o
 OBJS-$(CONFIG_VP56DSP)                 += vp56dsp.o
 OBJS-$(CONFIG_VP8DSP)                  += vp8dsp.o
 OBJS-$(CONFIG_V4L2_M2M)                += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
-OBJS-$(CONFIG_V4L2_REQUEST)            += v4l2_request.o v4l2_request_probe.o
+OBJS-$(CONFIG_V4L2_REQUEST)            += v4l2_request.o v4l2_request_probe.o v4l2_request_decode.o
 OBJS-$(CONFIG_WMA_FREQS)               += wma_freqs.o
 OBJS-$(CONFIG_WMV2DSP)                 += wmv2dsp.o
 
diff --git a/libavcodec/hwconfig.h b/libavcodec/hwconfig.h
index ee29ca631d..159064a1f1 100644
--- a/libavcodec/hwconfig.h
+++ b/libavcodec/hwconfig.h
@@ -79,6 +79,8 @@ void ff_hwaccel_uninit(AVCodecContext *avctx);
     HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD,  NONE,         ff_ ## codec ## _d3d11va_hwaccel)
 #define HWACCEL_D3D12VA(codec) \
     HW_CONFIG_HWACCEL(1, 1, 0, D3D12,        D3D12VA,      ff_ ## codec ## _d3d12va_hwaccel)
+#define HWACCEL_V4L2REQUEST(codec) \
+    HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME,    V4L2REQUEST,  ff_ ## codec ## _v4l2request_hwaccel)
 
 #define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \
     &(const AVCodecHWConfigInternal) { \
diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c
index 6f44c5d826..aae719ead6 100644
--- a/libavcodec/v4l2_request.c
+++ b/libavcodec/v4l2_request.c
@@ -303,7 +303,8 @@ int ff_v4l2_request_uninit(AVCodecContext *avctx)
     V4L2RequestContext *ctx = v4l2_request_context(avctx);
 
     if (ctx->video_fd >= 0) {
-        // TODO: Flush and wait on all pending requests
+        // Flush and wait on all pending requests
+        ff_v4l2_request_flush(avctx);
 
         // Stop output queue
         if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) {
diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h
index 621caaf28c..62248feb48 100644
--- a/libavcodec/v4l2_request.h
+++ b/libavcodec/v4l2_request.h
@@ -81,4 +81,27 @@ int ff_v4l2_request_init(AVCodecContext *avctx,
                          uint32_t pixelformat, uint32_t buffersize,
                          struct v4l2_ext_control *control, int count);
 
+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame);
+
+int ff_v4l2_request_append_output(AVCodecContext *avctx,
+                                  V4L2RequestPictureContext *pic,
+                                  const uint8_t *data, uint32_t size);
+
+int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
+                                 V4L2RequestPictureContext *pic,
+                                 struct v4l2_ext_control *control, int count,
+                                 bool first_slice, bool last_slice);
+
+int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
+                                 V4L2RequestPictureContext *pic,
+                                 struct v4l2_ext_control *control, int count);
+
+int ff_v4l2_request_reset_picture(AVCodecContext *avctx,
+                                  V4L2RequestPictureContext *pic);
+
+int ff_v4l2_request_start_frame(AVCodecContext *avctx,
+                                V4L2RequestPictureContext *pic, AVFrame *frame);
+
+void ff_v4l2_request_flush(AVCodecContext *avctx);
+
 #endif /* AVCODEC_V4L2_REQUEST_H */
diff --git a/libavcodec/v4l2_request_decode.c b/libavcodec/v4l2_request_decode.c
new file mode 100644
index 0000000000..8e7e241e36
--- /dev/null
+++ b/libavcodec/v4l2_request_decode.c
@@ -0,0 +1,459 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <poll.h>
+#include <sys/ioctl.h>
+
+#include "decode.h"
+#include "v4l2_request_internal.h"
+
+#define INPUT_BUFFER_PADDING_SIZE   (AV_INPUT_BUFFER_PADDING_SIZE * 4)
+
+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame)
+{
+    V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
+
+    /*
+     * The capture buffer index is used as a base for V4L2 frame reference.
+     * This works because frames are decoded into a capture buffer that is
+     * closely tied to an AVFrame.
+     */
+    return desc ? v4l2_timeval_to_ns(&desc->capture.buffer.timestamp) : 0;
+}
+
+static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, int request_fd,
+                                     V4L2RequestBuffer *buf, uint32_t flags)
+{
+    struct v4l2_plane planes[1] = {};
+    struct v4l2_buffer buffer = {
+        .index = buf->index,
+        .type = buf->buffer.type,
+        .memory = buf->buffer.memory,
+        .timestamp = buf->buffer.timestamp,
+        .bytesused = buf->used,
+        .request_fd = request_fd,
+        .flags = ((request_fd >= 0) ? V4L2_BUF_FLAG_REQUEST_FD : 0) | flags,
+    };
+
+    if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
+        planes[0].bytesused = buf->used;
+        buffer.bytesused = 0;
+        buffer.length = 1;
+        buffer.m.planes = planes;
+    }
+
+    // Queue the buffer
+    if (ioctl(ctx->video_fd, VIDIOC_QBUF, &buffer) < 0)
+        return AVERROR(errno);
+
+    // Mark the buffer as queued
+    if (V4L2_TYPE_IS_OUTPUT(buffer.type))
+        atomic_fetch_or(&ctx->queued_output, 1 << buffer.index);
+    else
+        atomic_fetch_or(&ctx->queued_capture, 1 << buffer.index);
+
+    return 0;
+}
+
+static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx,
+                                       enum v4l2_buf_type type)
+{
+    struct v4l2_plane planes[1] = {};
+    struct v4l2_buffer buffer = {
+        .type = type,
+        .memory = V4L2_MEMORY_MMAP,
+    };
+
+    if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
+        buffer.length = 1;
+        buffer.m.planes = planes;
+    }
+
+    // Dequeue next completed buffer
+    if (ioctl(ctx->video_fd, VIDIOC_DQBUF, &buffer) < 0)
+        return AVERROR(errno);
+
+    // Mark the buffer as dequeued
+    if (V4L2_TYPE_IS_OUTPUT(buffer.type))
+        atomic_fetch_and(&ctx->queued_output, ~(1 << buffer.index));
+    else
+        atomic_fetch_and(&ctx->queued_capture, ~(1 << buffer.index));
+
+    return 0;
+}
+
+static inline int v4l2_request_dequeue_completed_buffers(V4L2RequestContext *ctx,
+                                                         enum v4l2_buf_type type)
+{
+    int ret;
+
+    do {
+        ret = v4l2_request_dequeue_buffer(ctx, type);
+    } while (!ret);
+
+    return ret;
+}
+
+static int v4l2_request_wait_on_capture(V4L2RequestContext *ctx,
+                                        V4L2RequestBuffer *capture)
+{
+    struct pollfd pollfd = {
+        .fd = ctx->video_fd,
+        .events = POLLIN,
+    };
+
+    ff_mutex_lock(&ctx->mutex);
+
+    // Dequeue all completed capture buffers
+    if (atomic_load(&ctx->queued_capture))
+        v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
+
+    // Wait on the specific capture buffer, when needed
+    while (atomic_load(&ctx->queued_capture) & (1 << capture->index)) {
+        int ret = poll(&pollfd, 1, 2000);
+        if (ret <= 0)
+            goto fail;
+
+        ret = v4l2_request_dequeue_buffer(ctx, ctx->format.type);
+        if (ret < 0 && ret != AVERROR(EAGAIN))
+            goto fail;
+    }
+
+    ff_mutex_unlock(&ctx->mutex);
+    return 0;
+
+fail:
+    ff_mutex_unlock(&ctx->mutex);
+    av_log(ctx, AV_LOG_ERROR, "Failed waiting on capture buffer %d\n",
+           capture->index);
+    return AVERROR(EINVAL);
+}
+
+static V4L2RequestBuffer *v4l2_request_next_output(V4L2RequestContext *ctx)
+{
+    int index;
+    V4L2RequestBuffer *output;
+    struct pollfd pollfd = {
+        .fd = ctx->video_fd,
+        .events = POLLOUT,
+    };
+
+    ff_mutex_lock(&ctx->mutex);
+
+    // Use next output buffer in the circular queue
+    index = atomic_load(&ctx->next_output);
+    output = &ctx->output[index];
+    atomic_store(&ctx->next_output, (index + 1) % FF_ARRAY_ELEMS(ctx->output));
+
+    // Dequeue all completed output buffers
+    if (atomic_load(&ctx->queued_output))
+        v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
+
+    // Wait on the specific output buffer, when needed
+    while (atomic_load(&ctx->queued_output) & (1 << output->index)) {
+        int ret = poll(&pollfd, 1, 2000);
+        if (ret <= 0)
+            goto fail;
+
+        ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
+        if (ret < 0 && ret != AVERROR(EAGAIN))
+            goto fail;
+    }
+
+    ff_mutex_unlock(&ctx->mutex);
+
+    // Reset used state
+    output->used = 0;
+
+    return output;
+
+fail:
+    ff_mutex_unlock(&ctx->mutex);
+    av_log(ctx, AV_LOG_ERROR, "Failed waiting on output buffer %d\n",
+           output->index);
+    return NULL;
+}
+
+static int v4l2_request_wait_on_request(V4L2RequestContext *ctx,
+                                        V4L2RequestBuffer *output)
+{
+    struct pollfd pollfd = {
+        .fd = output->fd,
+        .events = POLLPRI,
+    };
+
+    // Wait on the specific request to complete, when needed
+    while (atomic_load(&ctx->queued_request) & (1 << output->index)) {
+        int ret = poll(&pollfd, 1, 2000);
+        if (ret <= 0)
+            break;
+
+        // Mark request as dequeued
+        if (pollfd.revents & (POLLPRI | POLLERR)) {
+            atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
+            break;
+        }
+    }
+
+    // Reinit the request object
+    if (ioctl(output->fd, MEDIA_REQUEST_IOC_REINIT, NULL) < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to reinit request object %d: %s (%d)\n",
+               output->fd, strerror(errno), errno);
+        return AVERROR(errno);
+    }
+
+    // Ensure request is marked as dequeued
+    atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
+
+    return 0;
+}
+
+int ff_v4l2_request_append_output(AVCodecContext *avctx,
+                                  V4L2RequestPictureContext *pic,
+                                  const uint8_t *data, uint32_t size)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+
+    // Append data to output buffer and ensure there is enough space for padding
+    if (pic->output->used + size + INPUT_BUFFER_PADDING_SIZE <= pic->output->size) {
+        memcpy(pic->output->addr + pic->output->used, data, size);
+        pic->output->used += size;
+        return 0;
+    } else {
+        av_log(ctx, AV_LOG_ERROR,
+               "Failed to append %u bytes data to output buffer %d (%u of %u used)\n",
+               size, pic->output->index, pic->output->used, pic->output->size);
+        return AVERROR(ENOMEM);
+    }
+}
+
+static int v4l2_request_queue_decode(AVCodecContext *avctx,
+                                     V4L2RequestPictureContext *pic,
+                                     struct v4l2_ext_control *control, int count,
+                                     bool first_slice, bool last_slice)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    uint32_t flags;
+    int ret;
+
+    if (first_slice) {
+        /*
+         * Wait on dequeue of the target capture buffer, when needed. Otherwise
+         * V4L2 decoder may use a different capture buffer than hwaccel expects.
+         *
+         * Normally decoding has already completed when a capture buffer is
+         * reused so this is more or less a no-op, however in some situations
+         * FFmpeg may reuse an AVFrame early, i.e. when no output frame was
+         * produced prior time, and a syncronization is necessary.
+         */
+        ret = v4l2_request_wait_on_capture(ctx, pic->capture);
+        if (ret < 0)
+            return ret;
+    }
+
+    ff_mutex_lock(&ctx->mutex);
+
+    /*
+     * The output buffer tied to prior use of current request object can
+     * independently be dequeued before the full decode request has been
+     * completed. This may happen when a decoder use multi stage decoding,
+     * e.g. rpivid. In such case we can start reusing the output buffer,
+     * however we must wait on the prior request to fully complete before we
+     * can reuse the request object, and a syncronization is necessary.
+     */
+    ret = v4l2_request_wait_on_request(ctx, pic->output);
+    if (ret < 0)
+        goto fail;
+
+    /*
+     * Dequeue any completed output buffers, this is strictly not necessary,
+     * however if a syncronization was necessary for the capture and/or request
+     * there is more than likely one or more output buffers that can be dequeued.
+     */
+    if (atomic_load(&ctx->queued_output))
+        v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
+
+    // Set codec controls for current request
+    ret = ff_v4l2_request_set_request_controls(ctx, pic->output->fd, control, count);
+    if (ret < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s) for request %d: %s (%d)\n",
+               count, pic->output->fd, strerror(errno), errno);
+        goto fail;
+    }
+
+    // Ensure there is zero padding at the end of bitstream data
+    memset(pic->output->addr + pic->output->used, 0, INPUT_BUFFER_PADDING_SIZE);
+
+    // Use timestamp of the capture buffer for V4L2 frame reference
+    pic->output->buffer.timestamp = pic->capture->buffer.timestamp;
+
+    /*
+     * Queue the output buffer of current request. The capture buffer may be
+     * hold by the V4L2 decoder unless this is the last slice of a frame.
+     */
+    flags = last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
+    ret = v4l2_request_queue_buffer(ctx, pic->output->fd, pic->output, flags);
+    if (ret < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to queue output buffer %d for request %d: %s (%d)\n",
+               pic->output->index, pic->output->fd, strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    if (first_slice) {
+        /*
+         * Queue the target capture buffer, hwaccel expect and depend on that
+         * this specific capture buffer will be used as decode target for
+         * current request, otherwise frames may be output in wrong order or
+         * wrong capture buffer could get used as a reference frame.
+         */
+        ret = v4l2_request_queue_buffer(ctx, -1, pic->capture, 0);
+        if (ret < 0) {
+            av_log(ctx, AV_LOG_ERROR, "Failed to queue capture buffer %d for request %d: %s (%d)\n",
+                   pic->capture->index, pic->output->fd, strerror(errno), errno);
+            ret = AVERROR(errno);
+            goto fail;
+        }
+    }
+
+    // Queue current request
+    ret = ioctl(pic->output->fd, MEDIA_REQUEST_IOC_QUEUE, NULL);
+    if (ret < 0) {
+        av_log(ctx, AV_LOG_ERROR, "Failed to queue request object %d: %s (%d)\n",
+               pic->output->fd, strerror(errno), errno);
+        ret = AVERROR(errno);
+        goto fail;
+    }
+
+    // Mark current request as queued
+    atomic_fetch_or(&ctx->queued_request, 1 << pic->output->index);
+
+    ret = 0;
+fail:
+    ff_mutex_unlock(&ctx->mutex);
+    return ret;
+}
+
+int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
+                                 V4L2RequestPictureContext *pic,
+                                 struct v4l2_ext_control *control, int count,
+                                 bool first_slice, bool last_slice)
+{
+    /*
+     * Fallback to queue each slice as a full frame when holding capture
+     * buffers is not supported by the driver.
+     */
+    if ((pic->output->capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) !=
+         V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF)
+        return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
+
+    return v4l2_request_queue_decode(avctx, pic, control, count,
+                                     first_slice, last_slice);
+}
+
+int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
+                                 V4L2RequestPictureContext *pic,
+                                 struct v4l2_ext_control *control, int count)
+{
+    return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
+}
+
+static int v4l2_request_post_process(void *logctx, AVFrame *frame)
+{
+    V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
+    FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
+    V4L2RequestContext *ctx = fdd->hwaccel_priv;
+
+    // Wait on capture buffer before returning the frame to application
+    return v4l2_request_wait_on_capture(ctx, &desc->capture);
+}
+
+int ff_v4l2_request_reset_picture(AVCodecContext *avctx, V4L2RequestPictureContext *pic)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+
+    // Get and wait on next output buffer from circular queue
+    pic->output = v4l2_request_next_output(ctx);
+    if (!pic->output)
+        return AVERROR(EINVAL);
+
+    return 0;
+}
+
+int ff_v4l2_request_start_frame(AVCodecContext *avctx,
+                                V4L2RequestPictureContext *pic,
+                                AVFrame *frame)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
+    FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
+    int ret;
+
+    // Get next output buffer from circular queue
+    ret = ff_v4l2_request_reset_picture(avctx, pic);
+    if (ret)
+        return ret;
+
+    // Ensure capture buffer is dequeued before reuse
+    ret = v4l2_request_wait_on_capture(ctx, &desc->capture);
+    if (ret)
+        return ret;
+
+    // Wait on capture buffer in post_process() before returning to application
+    fdd->hwaccel_priv = ctx;
+    fdd->post_process = v4l2_request_post_process;
+
+    // Capture buffer used for current frame
+    pic->capture = &desc->capture;
+
+    return 0;
+}
+
+void ff_v4l2_request_flush(AVCodecContext *avctx)
+{
+    V4L2RequestContext *ctx = v4l2_request_context(avctx);
+    struct pollfd pollfd = {
+        .fd = ctx->video_fd,
+        .events = POLLOUT,
+    };
+
+    ff_mutex_lock(&ctx->mutex);
+
+    // Dequeue all completed output buffers
+    if (atomic_load(&ctx->queued_output))
+        v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
+
+    // Wait on any remaining output buffer
+    while (atomic_load(&ctx->queued_output)) {
+        int ret = poll(&pollfd, 1, 2000);
+        if (ret <= 0)
+            break;
+
+        ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
+        if (ret < 0 && ret != AVERROR(EAGAIN))
+            break;
+    }
+
+    // Dequeue all completed capture buffers
+    if (atomic_load(&ctx->queued_capture))
+        v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
+
+    ff_mutex_unlock(&ctx->mutex);
+}

From 66cb225496620a953033096eb1013692be9cb931 Mon Sep 17 00:00:00 2001
From: Jonas Karlman <jonas@kwiboo.se>
Date: Tue, 6 Aug 2024 09:06:04 +0000
Subject: [PATCH 05/11] avcodec: Add V4L2 Request API mpeg2 hwaccel

Add a V4L2 Request API hwaccel for MPEG2.

Support for MPEG2 is enabled when Linux kernel headers declare the
control id V4L2_CID_STATELESS_MPEG2_SEQUENCE, added in v5.14.

This also change v4l2_request hwaccel to use autodetect in configure.

Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                       |   7 +-
 libavcodec/Makefile             |   1 +
 libavcodec/hwaccels.h           |   1 +
 libavcodec/mpeg12dec.c          |   6 ++
 libavcodec/v4l2_request_mpeg2.c | 176 ++++++++++++++++++++++++++++++++
 5 files changed, 189 insertions(+), 2 deletions(-)
 create mode 100644 libavcodec/v4l2_request_mpeg2.c

diff --git a/configure b/configure
index b74da32e8b..cbf0ee0ced 100755
--- a/configure
+++ b/configure
@@ -359,7 +359,7 @@ External library support:
   --enable-omx-rpi         enable OpenMAX IL code for Raspberry Pi [no]
   --enable-rkmpp           enable Rockchip Media Process Platform code [no]
   --disable-v4l2-m2m       disable V4L2 mem2mem code [autodetect]
-  --enable-v4l2-request    enable V4L2 Request API code [no]
+  --disable-v4l2-request   disable V4L2 Request API code [autodetect]
   --disable-vaapi          disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
   --disable-vdpau          disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
   --disable-videotoolbox   disable VideoToolbox code [autodetect]
@@ -2006,6 +2006,7 @@ HWACCEL_AUTODETECT_LIBRARY_LIST="
     videotoolbox
     vulkan
     v4l2_m2m
+    v4l2_request
 "
 
 # catchall list of things that require external libs to link
@@ -2027,7 +2028,6 @@ HWACCEL_LIBRARY_LIST="
     mmal
     omx
     opencl
-    v4l2_request
 "
 
 DOCUMENT_LIST="
@@ -3232,6 +3232,8 @@ mpeg2_dxva2_hwaccel_deps="dxva2"
 mpeg2_dxva2_hwaccel_select="mpeg2video_decoder"
 mpeg2_nvdec_hwaccel_deps="nvdec"
 mpeg2_nvdec_hwaccel_select="mpeg2video_decoder"
+mpeg2_v4l2request_hwaccel_deps="v4l2_request mpeg2_v4l2_request"
+mpeg2_v4l2request_hwaccel_select="mpeg2video_decoder"
 mpeg2_vaapi_hwaccel_deps="vaapi"
 mpeg2_vaapi_hwaccel_select="mpeg2video_decoder"
 mpeg2_vdpau_hwaccel_deps="vdpau"
@@ -7202,6 +7204,7 @@ if enabled v4l2_m2m; then
 fi
 
 if enabled v4l2_request; then
+    check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE"
     check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
     check_pkg_config libudev libudev libudev.h udev_new
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 6f2229b1b6..2c69b747b1 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1039,6 +1039,7 @@ OBJS-$(CONFIG_MPEG2_DXVA2_HWACCEL)        += dxva2_mpeg2.o
 OBJS-$(CONFIG_MPEG2_D3D12VA_HWACCEL)      += dxva2_mpeg2.o d3d12va_mpeg2.o
 OBJS-$(CONFIG_MPEG2_NVDEC_HWACCEL)        += nvdec_mpeg12.o
 OBJS-$(CONFIG_MPEG2_QSV_HWACCEL)          += qsvdec.o
+OBJS-$(CONFIG_MPEG2_V4L2REQUEST_HWACCEL)  += v4l2_request_mpeg2.o
 OBJS-$(CONFIG_MPEG2_VAAPI_HWACCEL)        += vaapi_mpeg2.o
 OBJS-$(CONFIG_MPEG2_VDPAU_HWACCEL)        += vdpau_mpeg12.o
 OBJS-$(CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index 5171e4c7d7..0cba7c71be 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -57,6 +57,7 @@ extern const struct FFHWAccel ff_mpeg2_d3d11va2_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_d3d12va_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_dxva2_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_nvdec_hwaccel;
+extern const struct FFHWAccel ff_mpeg2_v4l2request_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_vaapi_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_vdpau_hwaccel;
 extern const struct FFHWAccel ff_mpeg2_videotoolbox_hwaccel;
diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index 4f784611de..af8ffa5976 100644
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -839,6 +839,9 @@ static const enum AVPixelFormat mpeg2_hwaccel_pixfmt_list_420[] = {
 #endif
 #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL
     AV_PIX_FMT_VIDEOTOOLBOX,
+#endif
+#if CONFIG_MPEG2_V4L2REQUEST_HWACCEL
+    AV_PIX_FMT_DRM_PRIME,
 #endif
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_NONE
@@ -2681,6 +2684,9 @@ const FFCodec ff_mpeg2video_decoder = {
 #endif
 #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL
                         HWACCEL_VIDEOTOOLBOX(mpeg2),
+#endif
+#if CONFIG_MPEG2_V4L2REQUEST_HWACCEL
+                        HWACCEL_V4L2REQUEST(mpeg2),
 #endif
                         NULL
                     },
diff --git a/libavcodec/v4l2_request_mpeg2.c b/libavcodec/v4l2_request_mpeg2.c
new file mode 100644
index 0000000000..998c91355a
--- /dev/null
+++ b/libavcodec/v4l2_request_mpeg2.c
@@ -0,0 +1,176 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include "hwaccel_internal.h"
+#include "hwconfig.h"
+#include "mpegvideo.h"
+#include "v4l2_request.h"
+
+typedef struct V4L2RequestControlsMPEG2 {
+    V4L2RequestPictureContext pic;
+    struct v4l2_ctrl_mpeg2_sequence sequence;
+    struct v4l2_ctrl_mpeg2_picture picture;
+    struct v4l2_ctrl_mpeg2_quantisation quantisation;
+} V4L2RequestControlsMPEG2;
+
+static int v4l2_request_mpeg2_start_frame(AVCodecContext *avctx,
+                                          av_unused const uint8_t *buffer,
+                                          av_unused uint32_t size)
+{
+    const MpegEncContext *s = avctx->priv_data;
+    V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private;
+    int ret;
+
+    ret = ff_v4l2_request_start_frame(avctx, &controls->pic, s->cur_pic.ptr->f);
+    if (ret)
+        return ret;
+
+    controls->sequence = (struct v4l2_ctrl_mpeg2_sequence) {
+        /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence header */
+        .horizontal_size = s->width,
+        .vertical_size = s->height,
+        .vbv_buffer_size = controls->pic.output->size,
+
+        /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence extension */
+        .profile_and_level_indication = 0,
+        .chroma_format = s->chroma_format,
+    };
+
+    if (s->progressive_sequence)
+        controls->sequence.flags |= V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE;
+
+    controls->picture = (struct v4l2_ctrl_mpeg2_picture) {
+        /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture header */
+        .picture_coding_type = s->pict_type,
+
+        /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture coding extension */
+        .f_code[0][0] = s->mpeg_f_code[0][0],
+        .f_code[0][1] = s->mpeg_f_code[0][1],
+        .f_code[1][0] = s->mpeg_f_code[1][0],
+        .f_code[1][1] = s->mpeg_f_code[1][1],
+        .picture_structure = s->picture_structure,
+        .intra_dc_precision = s->intra_dc_precision,
+    };
+
+    if (s->top_field_first)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST;
+
+    if (s->frame_pred_frame_dct)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT;
+
+    if (s->concealment_motion_vectors)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV;
+
+    if (s->intra_vlc_format)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_INTRA_VLC;
+
+    if (s->q_scale_type)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE;
+
+    if (s->alternate_scan)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_ALT_SCAN;
+
+    if (s->repeat_first_field)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST;
+
+    if (s->progressive_frame)
+        controls->picture.flags |= V4L2_MPEG2_PIC_FLAG_PROGRESSIVE;
+
+    switch (s->pict_type) {
+    case AV_PICTURE_TYPE_B:
+        if (s->next_pic.ptr)
+            controls->picture.backward_ref_ts =
+                ff_v4l2_request_get_capture_timestamp(s->next_pic.ptr->f);
+        // fall-through
+    case AV_PICTURE_TYPE_P:
+        if (s->last_pic.ptr)
+            controls->picture.forward_ref_ts =
+                ff_v4l2_request_get_capture_timestamp(s->last_pic.ptr->f);
+    }
+
+    for (int i = 0; i < 64; i++) {
+        int n = s->idsp.idct_permutation[ff_zigzag_direct[i]];
+        controls->quantisation.intra_quantiser_matrix[i] = s->intra_matrix[n];
+        controls->quantisation.non_intra_quantiser_matrix[i] = s->inter_matrix[n];
+        controls->quantisation.chroma_intra_quantiser_matrix[i] = s->chroma_intra_matrix[n];
+        controls->quantisation.chroma_non_intra_quantiser_matrix[i] = s->chroma_inter_matrix[n];
+    }
+
+    return 0;
+}
+
+static int v4l2_request_mpeg2_decode_slice(AVCodecContext *avctx,
+                                           const uint8_t *buffer, uint32_t size)
+{
+    const MpegEncContext *s = avctx->priv_data;
+    V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private;
+
+    return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size);
+}
+
+static int v4l2_request_mpeg2_end_frame(AVCodecContext *avctx)
+{
+    const MpegEncContext *s = avctx->priv_data;
+    V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE,
+            .ptr = &controls->sequence,
+            .size = sizeof(controls->sequence),
+        },
+        {
+            .id = V4L2_CID_STATELESS_MPEG2_PICTURE,
+            .ptr = &controls->picture,
+            .size = sizeof(controls->picture),
+        },
+        {
+            .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION,
+            .ptr = &controls->quantisation,
+            .size = sizeof(controls->quantisation),
+        },
+    };
+
+    return ff_v4l2_request_decode_frame(avctx, &controls->pic,
+                                        control, FF_ARRAY_ELEMS(control));
+}
+
+static int v4l2_request_mpeg2_init(AVCodecContext *avctx)
+{
+    return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_MPEG2_SLICE,
+                                1024 * 1024,
+                                NULL, 0);
+}
+
+const FFHWAccel ff_mpeg2_v4l2request_hwaccel = {
+    .p.name             = "mpeg2_v4l2request",
+    .p.type             = AVMEDIA_TYPE_VIDEO,
+    .p.id               = AV_CODEC_ID_MPEG2VIDEO,
+    .p.pix_fmt          = AV_PIX_FMT_DRM_PRIME,
+    .start_frame        = v4l2_request_mpeg2_start_frame,
+    .decode_slice       = v4l2_request_mpeg2_decode_slice,
+    .end_frame          = v4l2_request_mpeg2_end_frame,
+    .flush              = ff_v4l2_request_flush,
+    .frame_priv_data_size = sizeof(V4L2RequestControlsMPEG2),
+    .init               = v4l2_request_mpeg2_init,
+    .uninit             = ff_v4l2_request_uninit,
+    .priv_data_size     = sizeof(V4L2RequestContext),
+    .frame_params       = ff_v4l2_request_frame_params,
+};

From bccaf9b66ca4e1180c0d42bc95daf540e1e2ffb7 Mon Sep 17 00:00:00 2001
From: Boris Brezillon <boris.brezillon@collabora.com>
Date: Tue, 6 Aug 2024 09:06:05 +0000
Subject: [PATCH 06/11] avcodec/h264dec: add ref_pic_marking and pic_order_cnt
 bit_size to slice context

The V4L2_CID_STATELESS_H264_DECODE_PARAMS control require following:

- dec_ref_pic_marking_bit_size
  Size in bits of the dec_ref_pic_marking() syntax element.

- pic_order_cnt_bit_size
  Combined size in bits of the picture order count related syntax
  elements: pic_order_cnt_lsb, delta_pic_order_cnt_bottom,
  delta_pic_order_cnt0, and delta_pic_order_cnt1.

Save the bit sizes while parsing for later use in hwaccel, similar to
short/long_term_ref_pic_set_size in hevcdec.

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 libavcodec/h264_slice.c | 6 +++++-
 libavcodec/h264dec.h    | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index a66b75ca80..58a48f3fbe 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1690,7 +1690,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl,
     unsigned int slice_type, tmp, i;
     int field_pic_flag, bottom_field_flag;
     int first_slice = sl == h->slice_ctx && !h->current_slice;
-    int picture_structure;
+    int picture_structure, pos;
 
     if (first_slice)
         av_assert0(!h->setup_finished);
@@ -1781,6 +1781,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl,
 
     sl->poc_lsb = 0;
     sl->delta_poc_bottom = 0;
+    pos = get_bits_left(&sl->gb);
     if (sps->poc_type == 0) {
         sl->poc_lsb = get_bits(&sl->gb, sps->log2_max_poc_lsb);
 
@@ -1795,6 +1796,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl,
         if (pps->pic_order_present == 1 && picture_structure == PICT_FRAME)
             sl->delta_poc[1] = get_se_golomb(&sl->gb);
     }
+    sl->pic_order_cnt_bit_size = pos - get_bits_left(&sl->gb);
 
     sl->redundant_pic_count = 0;
     if (pps->redundant_pic_cnt_present)
@@ -1834,9 +1836,11 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl,
 
     sl->explicit_ref_marking = 0;
     if (nal->ref_idc) {
+        pos = get_bits_left(&sl->gb);
         ret = ff_h264_decode_ref_pic_marking(sl, &sl->gb, nal, h->avctx);
         if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE))
             return AVERROR_INVALIDDATA;
+        sl->ref_pic_marking_bit_size = pos - get_bits_left(&sl->gb);
     }
 
     if (sl->slice_type_nos != AV_PICTURE_TYPE_I && pps->cabac) {
diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h
index ccd7583bf4..71cbb0e0cc 100644
--- a/libavcodec/h264dec.h
+++ b/libavcodec/h264dec.h
@@ -324,6 +324,7 @@ typedef struct H264SliceContext {
     MMCO mmco[H264_MAX_MMCO_COUNT];
     int  nb_mmco;
     int explicit_ref_marking;
+    int ref_pic_marking_bit_size;
 
     int frame_num;
     int idr_pic_id;
@@ -332,6 +333,7 @@ typedef struct H264SliceContext {
     int delta_poc[2];
     int curr_pic_num;
     int max_pic_num;
+    int pic_order_cnt_bit_size;
 } H264SliceContext;
 
 /**

From e7674a31a7bc8b37a8ba4be0fc59074333a95b2d Mon Sep 17 00:00:00 2001
From: Jernej Skrabec <jernej.skrabec@gmail.com>
Date: Tue, 6 Aug 2024 09:06:06 +0000
Subject: [PATCH 07/11] avcodec: Add V4L2 Request API h264 hwaccel

Add a V4L2 Request API hwaccel for H.264, supporting both slice and
frame decoding modes.

Support for H.264 is enabled when Linux kernel headers declare the
control id V4L2_CID_STATELESS_H264_DECODE_MODE, added in v5.11.

Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Jonas Karlman <jonas@kwiboo.se>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                      |   3 +
 libavcodec/Makefile            |   1 +
 libavcodec/h264_slice.c        |   7 +
 libavcodec/h264dec.c           |   3 +
 libavcodec/hwaccels.h          |   1 +
 libavcodec/v4l2_request_h264.c | 522 +++++++++++++++++++++++++++++++++
 6 files changed, 537 insertions(+)
 create mode 100644 libavcodec/v4l2_request_h264.c

diff --git a/configure b/configure
index cbf0ee0ced..6866a61153 100755
--- a/configure
+++ b/configure
@@ -3186,6 +3186,8 @@ h264_dxva2_hwaccel_deps="dxva2"
 h264_dxva2_hwaccel_select="h264_decoder"
 h264_nvdec_hwaccel_deps="nvdec"
 h264_nvdec_hwaccel_select="h264_decoder"
+h264_v4l2request_hwaccel_deps="v4l2_request h264_v4l2_request"
+h264_v4l2request_hwaccel_select="h264_decoder"
 h264_vaapi_hwaccel_deps="vaapi"
 h264_vaapi_hwaccel_select="h264_decoder"
 h264_vdpau_hwaccel_deps="vdpau"
@@ -7204,6 +7206,7 @@ if enabled v4l2_m2m; then
 fi
 
 if enabled v4l2_request; then
+    check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE"
     check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE"
     check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 2c69b747b1..00bc1cf8f2 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1017,6 +1017,7 @@ OBJS-$(CONFIG_H264_DXVA2_HWACCEL)         += dxva2_h264.o
 OBJS-$(CONFIG_H264_D3D12VA_HWACCEL)       += dxva2_h264.o d3d12va_h264.o
 OBJS-$(CONFIG_H264_NVDEC_HWACCEL)         += nvdec_h264.o
 OBJS-$(CONFIG_H264_QSV_HWACCEL)           += qsvdec.o
+OBJS-$(CONFIG_H264_V4L2REQUEST_HWACCEL)   += v4l2_request_h264.o
 OBJS-$(CONFIG_H264_VAAPI_HWACCEL)         += vaapi_h264.o
 OBJS-$(CONFIG_H264_VDPAU_HWACCEL)         += vdpau_h264.o
 OBJS-$(CONFIG_H264_VIDEOTOOLBOX_HWACCEL)  += videotoolbox.o
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 58a48f3fbe..259a2c307a 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -781,6 +781,7 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
                      (CONFIG_H264_D3D11VA_HWACCEL * 2) + \
                      CONFIG_H264_D3D12VA_HWACCEL + \
                      CONFIG_H264_NVDEC_HWACCEL + \
+                     CONFIG_H264_V4L2REQUEST_HWACCEL + \
                      CONFIG_H264_VAAPI_HWACCEL + \
                      CONFIG_H264_VIDEOTOOLBOX_HWACCEL + \
                      CONFIG_H264_VDPAU_HWACCEL + \
@@ -806,6 +807,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
 #endif
 #if CONFIG_H264_VULKAN_HWACCEL
         *fmt++ = AV_PIX_FMT_VULKAN;
+#endif
+#if CONFIG_H264_V4L2REQUEST_HWACCEL
+        *fmt++ = AV_PIX_FMT_DRM_PRIME;
 #endif
         if (CHROMA444(h)) {
             if (h->avctx->colorspace == AVCOL_SPC_RGB) {
@@ -862,6 +866,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
 #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL
         if (h->avctx->colorspace != AVCOL_SPC_RGB)
             *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX;
+#endif
+#if CONFIG_H264_V4L2REQUEST_HWACCEL
+        *fmt++ = AV_PIX_FMT_DRM_PRIME;
 #endif
         if (CHROMA444(h)) {
             if (h->avctx->colorspace == AVCOL_SPC_RGB)
diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index 0154fe17b6..a5e89ebddd 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -1160,6 +1160,9 @@ const FFCodec ff_h264_decoder = {
 #endif
 #if CONFIG_H264_VULKAN_HWACCEL
                                HWACCEL_VULKAN(h264),
+#endif
+#if CONFIG_H264_V4L2REQUEST_HWACCEL
+                               HWACCEL_V4L2REQUEST(h264),
 #endif
                                NULL
                            },
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index 0cba7c71be..024e021d73 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -34,6 +34,7 @@ extern const struct FFHWAccel ff_h264_d3d11va2_hwaccel;
 extern const struct FFHWAccel ff_h264_d3d12va_hwaccel;
 extern const struct FFHWAccel ff_h264_dxva2_hwaccel;
 extern const struct FFHWAccel ff_h264_nvdec_hwaccel;
+extern const struct FFHWAccel ff_h264_v4l2request_hwaccel;
 extern const struct FFHWAccel ff_h264_vaapi_hwaccel;
 extern const struct FFHWAccel ff_h264_vdpau_hwaccel;
 extern const struct FFHWAccel ff_h264_videotoolbox_hwaccel;
diff --git a/libavcodec/v4l2_request_h264.c b/libavcodec/v4l2_request_h264.c
new file mode 100644
index 0000000000..5affe309ac
--- /dev/null
+++ b/libavcodec/v4l2_request_h264.c
@@ -0,0 +1,522 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include "h264dec.h"
+#include "hwaccel_internal.h"
+#include "hwconfig.h"
+#include "internal.h"
+#include "v4l2_request.h"
+
+typedef struct V4L2RequestContextH264 {
+    V4L2RequestContext base;
+    enum v4l2_stateless_h264_decode_mode decode_mode;
+    enum v4l2_stateless_h264_start_code start_code;
+} V4L2RequestContextH264;
+
+typedef struct V4L2RequestControlsH264 {
+    V4L2RequestPictureContext pic;
+    struct v4l2_ctrl_h264_sps sps;
+    struct v4l2_ctrl_h264_pps pps;
+    struct v4l2_ctrl_h264_scaling_matrix scaling_matrix;
+    struct v4l2_ctrl_h264_decode_params decode_params;
+    struct v4l2_ctrl_h264_slice_params slice_params;
+    struct v4l2_ctrl_h264_pred_weights pred_weights;
+    bool pred_weights_required;
+    bool first_slice;
+    int num_slices;
+} V4L2RequestControlsH264;
+
+static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 };
+
+static void fill_weight_factors(struct v4l2_h264_weight_factors *weight_factors,
+                                int list, const H264SliceContext *sl)
+{
+    for (int i = 0; i < sl->ref_count[list]; i++) {
+        if (sl->pwt.luma_weight_flag[list]) {
+            weight_factors->luma_weight[i] = sl->pwt.luma_weight[i][list][0];
+            weight_factors->luma_offset[i] = sl->pwt.luma_weight[i][list][1];
+        } else {
+            weight_factors->luma_weight[i] = 1 << sl->pwt.luma_log2_weight_denom;
+            weight_factors->luma_offset[i] = 0;
+        }
+        for (int j = 0; j < 2; j++) {
+            if (sl->pwt.chroma_weight_flag[list]) {
+                weight_factors->chroma_weight[i][j] = sl->pwt.chroma_weight[i][list][j][0];
+                weight_factors->chroma_offset[i][j] = sl->pwt.chroma_weight[i][list][j][1];
+            } else {
+                weight_factors->chroma_weight[i][j] = 1 << sl->pwt.chroma_log2_weight_denom;
+                weight_factors->chroma_offset[i][j] = 0;
+            }
+        }
+    }
+}
+
+static void fill_dpb_entry(struct v4l2_h264_dpb_entry *entry,
+                           const H264Picture *pic, int long_idx)
+{
+    entry->reference_ts = ff_v4l2_request_get_capture_timestamp(pic->f);
+    entry->pic_num = pic->pic_id;
+    entry->frame_num = pic->long_ref ? long_idx : pic->frame_num;
+    entry->fields = pic->reference & V4L2_H264_FRAME_REF;
+    entry->flags = V4L2_H264_DPB_ENTRY_FLAG_VALID;
+    if (entry->fields)
+        entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE;
+    if (pic->long_ref)
+        entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM;
+    if (pic->field_picture)
+        entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD;
+    if (pic->field_poc[0] != INT_MAX)
+        entry->top_field_order_cnt = pic->field_poc[0];
+    if (pic->field_poc[1] != INT_MAX)
+        entry->bottom_field_order_cnt = pic->field_poc[1];
+}
+
+static void fill_dpb(struct v4l2_ctrl_h264_decode_params *decode_params,
+                     const H264Context *h)
+{
+    int entries = 0;
+
+    for (int i = 0; i < h->short_ref_count; i++) {
+        const H264Picture *pic = h->short_ref[i];
+        if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX))
+            fill_dpb_entry(&decode_params->dpb[entries++], pic, pic->pic_id);
+    }
+
+    if (!h->long_ref_count)
+        return;
+
+    for (int i = 0; i < FF_ARRAY_ELEMS(h->long_ref); i++) {
+        const H264Picture *pic = h->long_ref[i];
+        if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX))
+            fill_dpb_entry(&decode_params->dpb[entries++], pic, i);
+    }
+}
+
+static void fill_ref_list(struct v4l2_h264_reference *reference,
+                          struct v4l2_ctrl_h264_decode_params *decode_params,
+                          const H264Ref *ref)
+{
+    uint64_t timestamp;
+
+    if (!ref->parent)
+        return;
+
+    timestamp = ff_v4l2_request_get_capture_timestamp(ref->parent->f);
+
+    for (uint8_t i = 0; i < FF_ARRAY_ELEMS(decode_params->dpb); i++) {
+        struct v4l2_h264_dpb_entry *entry = &decode_params->dpb[i];
+        if ((entry->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) &&
+            entry->reference_ts == timestamp) {
+            reference->fields = ref->reference & V4L2_H264_FRAME_REF;
+            reference->index = i;
+            return;
+        }
+    }
+}
+
+static void fill_sps(struct v4l2_ctrl_h264_sps *ctrl, const H264Context *h)
+{
+    const SPS *sps = h->ps.sps;
+
+    *ctrl = (struct v4l2_ctrl_h264_sps) {
+        .profile_idc = sps->profile_idc,
+        .constraint_set_flags = sps->constraint_set_flags,
+        .level_idc = sps->level_idc,
+        .seq_parameter_set_id = sps->sps_id,
+        .chroma_format_idc = sps->chroma_format_idc,
+        .bit_depth_luma_minus8 = sps->bit_depth_luma - 8,
+        .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
+        .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4,
+        .pic_order_cnt_type = sps->poc_type,
+        .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4,
+        .max_num_ref_frames = sps->ref_frame_count,
+        .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length,
+        .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
+        .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
+        .pic_width_in_mbs_minus1 = h->mb_width - 1,
+        .pic_height_in_map_units_minus1 = sps->frame_mbs_only_flag ?
+                                          h->mb_height - 1 : h->mb_height / 2 - 1,
+    };
+
+    if (sps->poc_cycle_length > 0 && sps->poc_cycle_length <= 255)
+        memcpy(ctrl->offset_for_ref_frame, sps->offset_for_ref_frame,
+               sps->poc_cycle_length * sizeof(ctrl->offset_for_ref_frame[0]));
+
+    if (sps->residual_color_transform_flag)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE;
+
+    if (sps->transform_bypass)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS;
+
+    if (sps->delta_pic_order_always_zero_flag)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO;
+
+    if (sps->gaps_in_frame_num_allowed_flag)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED;
+
+    if (sps->frame_mbs_only_flag)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY;
+
+    if (sps->mb_aff)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD;
+
+    if (sps->direct_8x8_inference_flag)
+        ctrl->flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE;
+}
+
+static void fill_pps(struct v4l2_ctrl_h264_pps *ctrl, const H264Context *h)
+{
+    const SPS *sps = h->ps.sps;
+    const PPS *pps = h->ps.pps;
+    const H264SliceContext *sl = &h->slice_ctx[0];
+    int qp_bd_offset = 6 * (sps->bit_depth_luma - 8);
+
+    *ctrl = (struct v4l2_ctrl_h264_pps) {
+        .pic_parameter_set_id = sl->pps_id,
+        .seq_parameter_set_id = pps->sps_id,
+        .num_slice_groups_minus1 = pps->slice_group_count - 1,
+        .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1,
+        .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1,
+        .weighted_bipred_idc = pps->weighted_bipred_idc,
+        .pic_init_qp_minus26 = pps->init_qp - 26 - qp_bd_offset,
+        .pic_init_qs_minus26 = pps->init_qs - 26 - qp_bd_offset,
+        .chroma_qp_index_offset = pps->chroma_qp_index_offset[0],
+        .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1],
+    };
+
+    if (pps->cabac)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE;
+
+    if (pps->pic_order_present)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT;
+
+    if (pps->weighted_pred)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED;
+
+    if (pps->deblocking_filter_parameters_present)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT;
+
+    if (pps->constrained_intra_pred)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED;
+
+    if (pps->redundant_pic_cnt_present)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT;
+
+    if (pps->transform_8x8_mode)
+        ctrl->flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE;
+
+    /* FFmpeg always provide a scaling matrix */
+    ctrl->flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT;
+}
+
+static int v4l2_request_h264_start_frame(AVCodecContext *avctx,
+                                         av_unused const uint8_t *buffer,
+                                         av_unused uint32_t size)
+{
+    const H264Context *h = avctx->priv_data;
+    const PPS *pps = h->ps.pps;
+    const SPS *sps = h->ps.sps;
+    const H264SliceContext *sl = &h->slice_ctx[0];
+    V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private;
+    int ret;
+
+    ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_pic_ptr->f);
+    if (ret)
+        return ret;
+
+    fill_sps(&controls->sps, h);
+    fill_pps(&controls->pps, h);
+
+    memcpy(controls->scaling_matrix.scaling_list_4x4, pps->scaling_matrix4,
+           sizeof(controls->scaling_matrix.scaling_list_4x4));
+    memcpy(controls->scaling_matrix.scaling_list_8x8[0], pps->scaling_matrix8[0],
+           sizeof(controls->scaling_matrix.scaling_list_8x8[0]));
+    memcpy(controls->scaling_matrix.scaling_list_8x8[1], pps->scaling_matrix8[3],
+           sizeof(controls->scaling_matrix.scaling_list_8x8[1]));
+
+    if (sps->chroma_format_idc == 3) {
+        memcpy(controls->scaling_matrix.scaling_list_8x8[2], pps->scaling_matrix8[1],
+               sizeof(controls->scaling_matrix.scaling_list_8x8[2]));
+        memcpy(controls->scaling_matrix.scaling_list_8x8[3], pps->scaling_matrix8[4],
+               sizeof(controls->scaling_matrix.scaling_list_8x8[3]));
+        memcpy(controls->scaling_matrix.scaling_list_8x8[4], pps->scaling_matrix8[2],
+               sizeof(controls->scaling_matrix.scaling_list_8x8[4]));
+        memcpy(controls->scaling_matrix.scaling_list_8x8[5], pps->scaling_matrix8[5],
+               sizeof(controls->scaling_matrix.scaling_list_8x8[5]));
+    }
+
+    controls->decode_params = (struct v4l2_ctrl_h264_decode_params) {
+        .nal_ref_idc = h->nal_ref_idc,
+        .frame_num = h->poc.frame_num,
+        .top_field_order_cnt = h->cur_pic_ptr->field_poc[0] != INT_MAX ?
+                               h->cur_pic_ptr->field_poc[0] : 0,
+        .bottom_field_order_cnt = h->cur_pic_ptr->field_poc[1] != INT_MAX ?
+                                  h->cur_pic_ptr->field_poc[1] : 0,
+        .idr_pic_id = sl->idr_pic_id,
+        .pic_order_cnt_lsb = sl->poc_lsb,
+        .delta_pic_order_cnt_bottom = sl->delta_poc_bottom,
+        .delta_pic_order_cnt0 = sl->delta_poc[0],
+        .delta_pic_order_cnt1 = sl->delta_poc[1],
+        /* Size in bits of dec_ref_pic_marking() syntax element. */
+        .dec_ref_pic_marking_bit_size = sl->ref_pic_marking_bit_size,
+        /* Size in bits of pic order count syntax. */
+        .pic_order_cnt_bit_size = sl->pic_order_cnt_bit_size,
+        .slice_group_change_cycle = 0, /* slice group not supported by FFmpeg */
+    };
+
+    if (h->picture_idr)
+        controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC;
+
+    if (FIELD_PICTURE(h))
+        controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC;
+
+    if (h->picture_structure == PICT_BOTTOM_FIELD)
+        controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD;
+
+#if defined(V4L2_H264_DECODE_PARAM_FLAG_PFRAME)
+    if (sl->slice_type_nos == AV_PICTURE_TYPE_P)
+        controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_PFRAME;
+#endif
+
+#if defined(V4L2_H264_DECODE_PARAM_FLAG_BFRAME)
+    if (sl->slice_type_nos == AV_PICTURE_TYPE_B)
+        controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BFRAME;
+#endif
+
+    fill_dpb(&controls->decode_params, h);
+
+    controls->first_slice = true;
+    controls->num_slices = 0;
+
+    return 0;
+}
+
+static int v4l2_request_h264_queue_decode(AVCodecContext *avctx, bool last_slice)
+{
+    const H264Context *h = avctx->priv_data;
+    V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data;
+    V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_H264_SPS,
+            .ptr = &controls->sps,
+            .size = sizeof(controls->sps),
+        },
+        {
+            .id = V4L2_CID_STATELESS_H264_PPS,
+            .ptr = &controls->pps,
+            .size = sizeof(controls->pps),
+        },
+        {
+            .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX,
+            .ptr = &controls->scaling_matrix,
+            .size = sizeof(controls->scaling_matrix),
+        },
+        {
+            .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS,
+            .ptr = &controls->decode_params,
+            .size = sizeof(controls->decode_params),
+        },
+        {
+            .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS,
+            .ptr = &controls->slice_params,
+            .size = sizeof(controls->slice_params),
+        },
+        {
+            .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS,
+            .ptr = &controls->pred_weights,
+            .size = sizeof(controls->pred_weights),
+        },
+    };
+
+    if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) {
+        int count = FF_ARRAY_ELEMS(control) - (controls->pred_weights_required ? 0 : 1);
+        return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count,
+                                            controls->first_slice, last_slice);
+    }
+
+    return ff_v4l2_request_decode_frame(avctx, &controls->pic,
+                                        control, FF_ARRAY_ELEMS(control) - 2);
+}
+
+static int v4l2_request_h264_decode_slice(AVCodecContext *avctx,
+                                          const uint8_t *buffer, uint32_t size)
+{
+    const H264Context *h = avctx->priv_data;
+    const PPS *pps = h->ps.pps;
+    const H264SliceContext *sl = &h->slice_ctx[0];
+    V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data;
+    V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private;
+    int i, ret, count;
+
+    if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED &&
+        controls->num_slices) {
+        ret = v4l2_request_h264_queue_decode(avctx, false);
+        if (ret)
+            return ret;
+
+        ff_v4l2_request_reset_picture(avctx, &controls->pic);
+        controls->first_slice = 0;
+    }
+
+    if (ctx->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B) {
+        ret = ff_v4l2_request_append_output(avctx, &controls->pic,
+                                            nalu_slice_start_code, 3);
+        if (ret)
+            return ret;
+    }
+
+    ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size);
+    if (ret)
+        return ret;
+
+    if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED)
+        return 0;
+
+    controls->slice_params = (struct v4l2_ctrl_h264_slice_params) {
+        /* Offset in bits to slice_data() from the beginning of this slice. */
+        .header_bit_size = get_bits_count(&sl->gb),
+
+        .first_mb_in_slice = sl->first_mb_addr,
+
+        .slice_type = ff_h264_get_slice_type(sl),
+        .colour_plane_id = 0, /* separate colour plane not supported by FFmpeg */
+        .redundant_pic_cnt = sl->redundant_pic_count,
+        .cabac_init_idc = sl->cabac_init_idc,
+        .slice_qp_delta = sl->qscale - pps->init_qp,
+        .slice_qs_delta = 0, /* not implemented by FFmpeg */
+        .disable_deblocking_filter_idc = sl->deblocking_filter < 2 ?
+                                         !sl->deblocking_filter :
+                                         sl->deblocking_filter,
+        .slice_alpha_c0_offset_div2 = sl->slice_alpha_c0_offset / 2,
+        .slice_beta_offset_div2 = sl->slice_beta_offset / 2,
+        .num_ref_idx_l0_active_minus1 = sl->list_count > 0 ? sl->ref_count[0] - 1 : 0,
+        .num_ref_idx_l1_active_minus1 = sl->list_count > 1 ? sl->ref_count[1] - 1 : 0,
+    };
+
+    if (sl->slice_type == AV_PICTURE_TYPE_B && sl->direct_spatial_mv_pred)
+        controls->slice_params.flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED;
+
+    /* V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH: not implemented by FFmpeg */
+
+    controls->pred_weights_required =
+        V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(&controls->pps, &controls->slice_params);
+    if (controls->pred_weights_required) {
+        controls->pred_weights.chroma_log2_weight_denom = sl->pwt.chroma_log2_weight_denom;
+        controls->pred_weights.luma_log2_weight_denom = sl->pwt.luma_log2_weight_denom;
+    }
+
+    count = sl->list_count > 0 ? sl->ref_count[0] : 0;
+    for (i = 0; i < count; i++)
+        fill_ref_list(&controls->slice_params.ref_pic_list0[i],
+                      &controls->decode_params, &sl->ref_list[0][i]);
+    if (count && controls->pred_weights_required)
+        fill_weight_factors(&controls->pred_weights.weight_factors[0], 0, sl);
+
+    count = sl->list_count > 1 ? sl->ref_count[1] : 0;
+    for (i = 0; i < count; i++)
+        fill_ref_list(&controls->slice_params.ref_pic_list1[i],
+                      &controls->decode_params, &sl->ref_list[1][i]);
+    if (count && controls->pred_weights_required)
+        fill_weight_factors(&controls->pred_weights.weight_factors[1], 1, sl);
+
+    controls->num_slices++;
+    return 0;
+}
+
+static int v4l2_request_h264_end_frame(AVCodecContext *avctx)
+{
+    return v4l2_request_h264_queue_decode(avctx, true);
+}
+
+static int v4l2_request_h264_post_probe(AVCodecContext *avctx)
+{
+    V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data;
+
+    struct v4l2_ext_control control[] = {
+        { .id = V4L2_CID_STATELESS_H264_DECODE_MODE, },
+        { .id = V4L2_CID_STATELESS_H264_START_CODE, },
+    };
+
+    ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx,
+                                            V4L2_CID_STATELESS_H264_DECODE_MODE);
+    if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED &&
+        ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED) {
+        av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n",
+               ctx->decode_mode);
+        return AVERROR(EINVAL);
+    }
+
+    ctx->start_code = ff_v4l2_request_query_control_default_value(avctx,
+                                            V4L2_CID_STATELESS_H264_START_CODE);
+    if (ctx->start_code != V4L2_STATELESS_H264_START_CODE_NONE &&
+        ctx->start_code != V4L2_STATELESS_H264_START_CODE_ANNEX_B) {
+        av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n",
+               ctx->start_code);
+        return AVERROR(EINVAL);
+    }
+
+    // TODO: check V4L2_CID_MPEG_VIDEO_H264_PROFILE control
+    // TODO: check V4L2_CID_MPEG_VIDEO_H264_LEVEL control
+
+    control[0].value = ctx->decode_mode;
+    control[1].value = ctx->start_code;
+
+    return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control));
+}
+
+static int v4l2_request_h264_init(AVCodecContext *avctx)
+{
+    V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data;
+    const H264Context *h = avctx->priv_data;
+    struct v4l2_ctrl_h264_sps sps;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_H264_SPS,
+            .ptr = &sps,
+            .size = sizeof(sps),
+        },
+    };
+
+    fill_sps(&sps, h);
+
+    ctx->base.post_probe = v4l2_request_h264_post_probe;
+    return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_H264_SLICE,
+                                4 * 1024 * 1024,
+                                control, FF_ARRAY_ELEMS(control));
+}
+
+const FFHWAccel ff_h264_v4l2request_hwaccel = {
+    .p.name             = "h264_v4l2request",
+    .p.type             = AVMEDIA_TYPE_VIDEO,
+    .p.id               = AV_CODEC_ID_H264,
+    .p.pix_fmt          = AV_PIX_FMT_DRM_PRIME,
+    .start_frame        = v4l2_request_h264_start_frame,
+    .decode_slice       = v4l2_request_h264_decode_slice,
+    .end_frame          = v4l2_request_h264_end_frame,
+    .flush              = ff_v4l2_request_flush,
+    .frame_priv_data_size = sizeof(V4L2RequestControlsH264),
+    .init               = v4l2_request_h264_init,
+    .uninit             = ff_v4l2_request_uninit,
+    .priv_data_size     = sizeof(V4L2RequestContextH264),
+    .frame_params       = ff_v4l2_request_frame_params,
+};

From 0b3c85ab94619e449026145a9795f2013732977c Mon Sep 17 00:00:00 2001
From: Jernej Skrabec <jernej.skrabec@gmail.com>
Date: Tue, 6 Aug 2024 09:06:07 +0000
Subject: [PATCH 08/11] avcodec: Add V4L2 Request API hevc hwaccel

Add a V4L2 Request API hwaccel for HEVC, supporting both slice and
frame decoding modes.

Support for HEVC is enabled when Linux kernel headers declare the
control id V4L2_CID_STATELESS_HEVC_SPS, added in v6.0.

Co-developed-by: Benjamin Gaignard <benjamin.gaignard@collabora.com>
Signed-off-by: Benjamin Gaignard <benjamin.gaignard@collabora.com>
Co-developed-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Alex Bee <knaerzche@gmail.com>
Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Jonas Karlman <jonas@kwiboo.se>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                      |   5 +
 libavcodec/Makefile            |   1 +
 libavcodec/hevc/hevcdec.c      |  10 +
 libavcodec/hwaccels.h          |   1 +
 libavcodec/v4l2_request_hevc.c | 745 +++++++++++++++++++++++++++++++++
 5 files changed, 762 insertions(+)
 create mode 100644 libavcodec/v4l2_request_hevc.c

diff --git a/configure b/configure
index 6866a61153..7a2901d960 100755
--- a/configure
+++ b/configure
@@ -2491,6 +2491,7 @@ TYPES_LIST="
     struct_sockaddr_sa_len
     struct_sockaddr_storage
     struct_stat_st_mtim_tv_nsec
+    struct_v4l2_ctrl_hevc_decode_params_num_delta_pocs_of_ref_rps_idx
     struct_v4l2_frmivalenum_discrete
     struct_mfxConfigInterface
 "
@@ -3206,6 +3207,8 @@ hevc_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_HEVC"
 hevc_dxva2_hwaccel_select="hevc_decoder"
 hevc_nvdec_hwaccel_deps="nvdec"
 hevc_nvdec_hwaccel_select="hevc_decoder"
+hevc_v4l2request_hwaccel_deps="v4l2_request hevc_v4l2_request"
+hevc_v4l2request_hwaccel_select="hevc_decoder"
 hevc_vaapi_hwaccel_deps="vaapi VAPictureParameterBufferHEVC"
 hevc_vaapi_hwaccel_select="hevc_decoder"
 hevc_vdpau_hwaccel_deps="vdpau VdpPictureInfoHEVC"
@@ -7207,10 +7210,12 @@ fi
 
 if enabled v4l2_request; then
     check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE"
+    check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS"
     check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE"
     check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
     check_pkg_config libudev libudev libudev.h udev_new
+    check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx
 fi
 
 check_headers sys/videoio.h
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 00bc1cf8f2..fa78523b76 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1027,6 +1027,7 @@ OBJS-$(CONFIG_HEVC_DXVA2_HWACCEL)         += dxva2_hevc.o
 OBJS-$(CONFIG_HEVC_D3D12VA_HWACCEL)       += dxva2_hevc.o d3d12va_hevc.o
 OBJS-$(CONFIG_HEVC_NVDEC_HWACCEL)         += nvdec_hevc.o
 OBJS-$(CONFIG_HEVC_QSV_HWACCEL)           += qsvdec.o
+OBJS-$(CONFIG_HEVC_V4L2REQUEST_HWACCEL)   += v4l2_request_hevc.o
 OBJS-$(CONFIG_HEVC_VAAPI_HWACCEL)         += vaapi_hevc.o h265_profile_level.o
 OBJS-$(CONFIG_HEVC_VDPAU_HWACCEL)         += vdpau_hevc.o h265_profile_level.o
 OBJS-$(CONFIG_HEVC_VULKAN_HWACCEL)        += vulkan_decode.o vulkan_hevc.o
diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 0dc24f82f8..4b14a931a2 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -527,6 +527,7 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
                      CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \
                      CONFIG_HEVC_D3D12VA_HWACCEL + \
                      CONFIG_HEVC_NVDEC_HWACCEL + \
+                     CONFIG_HEVC_V4L2REQUEST_HWACCEL + \
                      CONFIG_HEVC_VAAPI_HWACCEL + \
                      CONFIG_HEVC_VIDEOTOOLBOX_HWACCEL + \
                      CONFIG_HEVC_VDPAU_HWACCEL + \
@@ -561,6 +562,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
 #endif
 #if CONFIG_HEVC_VULKAN_HWACCEL
         *fmt++ = AV_PIX_FMT_VULKAN;
+#endif
+#if CONFIG_HEVC_V4L2REQUEST_HWACCEL
+        *fmt++ = AV_PIX_FMT_DRM_PRIME;
 #endif
         break;
     case AV_PIX_FMT_YUV420P10:
@@ -588,6 +592,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
 #endif
 #if CONFIG_HEVC_NVDEC_HWACCEL
         *fmt++ = AV_PIX_FMT_CUDA;
+#endif
+#if CONFIG_HEVC_V4L2REQUEST_HWACCEL
+        *fmt++ = AV_PIX_FMT_DRM_PRIME;
 #endif
         break;
     case AV_PIX_FMT_YUV444P:
@@ -4107,6 +4114,9 @@ const FFCodec ff_hevc_decoder = {
 #endif
 #if CONFIG_HEVC_VULKAN_HWACCEL
                                HWACCEL_VULKAN(hevc),
+#endif
+#if CONFIG_HEVC_V4L2REQUEST_HWACCEL
+                               HWACCEL_V4L2REQUEST(hevc),
 #endif
                                NULL
                            },
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index 024e021d73..ad145fc07c 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -44,6 +44,7 @@ extern const struct FFHWAccel ff_hevc_d3d11va2_hwaccel;
 extern const struct FFHWAccel ff_hevc_d3d12va_hwaccel;
 extern const struct FFHWAccel ff_hevc_dxva2_hwaccel;
 extern const struct FFHWAccel ff_hevc_nvdec_hwaccel;
+extern const struct FFHWAccel ff_hevc_v4l2request_hwaccel;
 extern const struct FFHWAccel ff_hevc_vaapi_hwaccel;
 extern const struct FFHWAccel ff_hevc_vdpau_hwaccel;
 extern const struct FFHWAccel ff_hevc_videotoolbox_hwaccel;
diff --git a/libavcodec/v4l2_request_hevc.c b/libavcodec/v4l2_request_hevc.c
new file mode 100644
index 0000000000..9fabe468b7
--- /dev/null
+++ b/libavcodec/v4l2_request_hevc.c
@@ -0,0 +1,745 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include "libavutil/mem.h"
+#include "hevc/hevcdec.h"
+#include "hwaccel_internal.h"
+#include "hwconfig.h"
+#include "internal.h"
+#include "v4l2_request.h"
+
+#define V4L2_HEVC_CONTROLS_MAX 6
+
+typedef struct V4L2RequestContextHEVC {
+    V4L2RequestContext base;
+    enum v4l2_stateless_hevc_decode_mode decode_mode;
+    enum v4l2_stateless_hevc_start_code start_code;
+    unsigned int max_slice_params;
+    unsigned int max_entry_point_offsets;
+    bool has_scaling_matrix;
+} V4L2RequestContextHEVC;
+
+typedef struct V4L2RequestControlsHEVC {
+    V4L2RequestPictureContext pic;
+    struct v4l2_ctrl_hevc_sps sps;
+    struct v4l2_ctrl_hevc_pps pps;
+    struct v4l2_ctrl_hevc_decode_params decode_params;
+    struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix;
+    struct v4l2_ctrl_hevc_slice_params slice_params;
+    struct v4l2_ctrl_hevc_slice_params *frame_slice_params;
+    unsigned int allocated_slice_params;
+    unsigned int num_slice_params;
+    uint32_t *entry_point_offsets;
+    unsigned int allocated_entry_point_offsets;
+    unsigned int num_entry_point_offsets;
+    bool first_slice;
+} V4L2RequestControlsHEVC;
+
+static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 };
+
+static void fill_pred_weight_table(struct v4l2_hevc_pred_weight_table *table,
+                                   const HEVCContext *h)
+{
+    int32_t luma_weight_denom, chroma_weight_denom;
+    const SliceHeader *sh = &h->sh;
+
+    if (sh->slice_type == HEVC_SLICE_I ||
+        (sh->slice_type == HEVC_SLICE_P && !h->pps->weighted_pred_flag) ||
+        (sh->slice_type == HEVC_SLICE_B && !h->pps->weighted_bipred_flag))
+        return;
+
+    table->luma_log2_weight_denom = sh->luma_log2_weight_denom;
+
+    if (h->pps->sps->chroma_format_idc)
+        table->delta_chroma_log2_weight_denom = sh->chroma_log2_weight_denom -
+                                                sh->luma_log2_weight_denom;
+
+    luma_weight_denom = (1 << sh->luma_log2_weight_denom);
+    chroma_weight_denom = (1 << sh->chroma_log2_weight_denom);
+
+    for (int i = 0; i < 15 && i < sh->nb_refs[L0]; i++) {
+        table->delta_luma_weight_l0[i] = sh->luma_weight_l0[i] - luma_weight_denom;
+        table->luma_offset_l0[i] = sh->luma_offset_l0[i];
+        table->delta_chroma_weight_l0[i][0] = sh->chroma_weight_l0[i][0] - chroma_weight_denom;
+        table->delta_chroma_weight_l0[i][1] = sh->chroma_weight_l0[i][1] - chroma_weight_denom;
+        table->chroma_offset_l0[i][0] = sh->chroma_offset_l0[i][0];
+        table->chroma_offset_l0[i][1] = sh->chroma_offset_l0[i][1];
+    }
+
+    if (sh->slice_type != HEVC_SLICE_B)
+        return;
+
+    for (int i = 0; i < 15 && i < sh->nb_refs[L1]; i++) {
+        table->delta_luma_weight_l1[i] = sh->luma_weight_l1[i] - luma_weight_denom;
+        table->luma_offset_l1[i] = sh->luma_offset_l1[i];
+        table->delta_chroma_weight_l1[i][0] = sh->chroma_weight_l1[i][0] - chroma_weight_denom;
+        table->delta_chroma_weight_l1[i][1] = sh->chroma_weight_l1[i][1] - chroma_weight_denom;
+        table->chroma_offset_l1[i][0] = sh->chroma_offset_l1[i][0];
+        table->chroma_offset_l1[i][1] = sh->chroma_offset_l1[i][1];
+    }
+}
+
+static uint8_t get_ref_pic_index(const HEVCContext *h, const HEVCFrame *frame,
+                                 struct v4l2_ctrl_hevc_decode_params *decode_params)
+{
+    uint64_t timestamp;
+
+    if (!frame || !frame->f)
+        return 0;
+
+    timestamp = ff_v4l2_request_get_capture_timestamp(frame->f);
+
+    for (uint8_t i = 0; i < decode_params->num_active_dpb_entries; i++) {
+        struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[i];
+        if (entry->timestamp == timestamp)
+            return i;
+    }
+
+    return 0;
+}
+
+static void fill_decode_params(struct v4l2_ctrl_hevc_decode_params *decode_params,
+                               const HEVCContext *h)
+{
+    const HEVCFrame *pic = h->cur_frame;
+    const HEVCLayerContext *l = &h->layers[h->cur_layer];
+    const SliceHeader *sh = &h->sh;
+    int i, entries = 0;
+
+    *decode_params = (struct v4l2_ctrl_hevc_decode_params) {
+        .pic_order_cnt_val = h->poc,
+        .short_term_ref_pic_set_size = sh->short_term_ref_pic_set_size,
+        .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size,
+        .num_poc_st_curr_before = h->rps[ST_CURR_BEF].nb_refs,
+        .num_poc_st_curr_after = h->rps[ST_CURR_AFT].nb_refs,
+        .num_poc_lt_curr = h->rps[LT_CURR].nb_refs,
+    };
+
+#if HAVE_STRUCT_V4L2_CTRL_HEVC_DECODE_PARAMS_NUM_DELTA_POCS_OF_REF_RPS_IDX
+    if (h->sh.short_term_ref_pic_set_sps_flag == 0 && h->sh.short_term_rps)
+        decode_params->num_delta_pocs_of_ref_rps_idx =
+                                h->sh.short_term_rps->rps_idx_num_delta_pocs;
+#endif
+
+    for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++) {
+        const HEVCFrame *frame = &l->DPB[i];
+        if (frame != pic &&
+            (frame->flags & (HEVC_FRAME_FLAG_LONG_REF | HEVC_FRAME_FLAG_SHORT_REF))) {
+            struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[entries++];
+
+            entry->timestamp = ff_v4l2_request_get_capture_timestamp(frame->f);
+            entry->field_pic = !!(frame->f->flags & AV_FRAME_FLAG_INTERLACED);
+            entry->flags = 0;
+            if (frame->flags & HEVC_FRAME_FLAG_LONG_REF)
+                entry->flags |= V4L2_HEVC_DPB_ENTRY_LONG_TERM_REFERENCE;
+
+            entry->pic_order_cnt_val = frame->poc;
+        }
+    }
+
+    decode_params->num_active_dpb_entries = entries;
+
+    if (IS_IRAP(h))
+        decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC;
+
+    if (IS_IDR(h))
+        decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC;
+
+    if (sh->no_output_of_prior_pics_flag)
+        decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR;
+
+    for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) {
+        decode_params->poc_st_curr_before[i] =
+            get_ref_pic_index(h, h->rps[ST_CURR_BEF].ref[i], decode_params);
+        decode_params->poc_st_curr_after[i] =
+            get_ref_pic_index(h, h->rps[ST_CURR_AFT].ref[i], decode_params);
+        decode_params->poc_lt_curr[i] =
+            get_ref_pic_index(h, h->rps[LT_CURR].ref[i], decode_params);
+    }
+}
+
+static int fill_slice_params(V4L2RequestControlsHEVC *controls, int slice,
+                             bool max_entry_point_offsets, const HEVCContext *h)
+{
+    struct v4l2_ctrl_hevc_slice_params *slice_params = &controls->frame_slice_params[slice];
+    struct v4l2_ctrl_hevc_decode_params *decode_params = &controls->decode_params;
+    const SliceHeader *sh = &h->sh;
+    RefPicList *rpl;
+    int i, offsets;
+
+    *slice_params = (struct v4l2_ctrl_hevc_slice_params) {
+        .bit_size = 0,
+        .data_byte_offset = controls->pic.output->used + sh->data_offset,
+        .num_entry_point_offsets = sh->num_entry_point_offsets,
+
+        /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */
+        .nal_unit_type = h->nal_unit_type,
+        .nuh_temporal_id_plus1 = h->temporal_id + 1,
+
+        /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */
+        .slice_type = sh->slice_type,
+        .colour_plane_id = sh->colour_plane_id,
+        .slice_pic_order_cnt = sh->poc,
+        .num_ref_idx_l0_active_minus1 = sh->nb_refs[L0] ? sh->nb_refs[L0] - 1 : 0,
+        .num_ref_idx_l1_active_minus1 = sh->nb_refs[L1] ? sh->nb_refs[L1] - 1 : 0,
+        .collocated_ref_idx = sh->slice_temporal_mvp_enabled_flag ?
+                              sh->collocated_ref_idx : 0,
+        .five_minus_max_num_merge_cand = sh->slice_type == HEVC_SLICE_I ?
+                                         0 : 5 - sh->max_num_merge_cand,
+        .slice_qp_delta = sh->slice_qp_delta,
+        .slice_cb_qp_offset = sh->slice_cb_qp_offset,
+        .slice_cr_qp_offset = sh->slice_cr_qp_offset,
+        .slice_act_y_qp_offset = 0,
+        .slice_act_cb_qp_offset = 0,
+        .slice_act_cr_qp_offset = 0,
+        .slice_beta_offset_div2 = sh->beta_offset / 2,
+        .slice_tc_offset_div2 = sh->tc_offset / 2,
+
+        /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture timing SEI message */
+        .pic_struct = h->sei.picture_timing.picture_struct,
+
+        /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */
+        .slice_segment_addr = sh->slice_segment_addr,
+        .short_term_ref_pic_set_size = sh->short_term_ref_pic_set_size,
+        .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size,
+    };
+
+    if (h->pps->pps_slice_act_qp_offsets_present_flag) {
+        slice_params->slice_act_y_qp_offset = sh->slice_act_y_qp_offset;
+        slice_params->slice_act_cb_qp_offset = sh->slice_act_cb_qp_offset;
+        slice_params->slice_act_cr_qp_offset = sh->slice_act_cr_qp_offset;
+    }
+
+    if (sh->slice_sample_adaptive_offset_flag[0])
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_SAO_LUMA;
+
+    if (sh->slice_sample_adaptive_offset_flag[1])
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_SAO_CHROMA;
+
+    if (sh->slice_temporal_mvp_enabled_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_TEMPORAL_MVP_ENABLED;
+
+    if (sh->mvd_l1_zero_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_MVD_L1_ZERO;
+
+    if (sh->cabac_init_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_CABAC_INIT;
+
+    if (sh->collocated_list == L0)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_COLLOCATED_FROM_L0;
+
+    if (sh->use_integer_mv_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_USE_INTEGER_MV;
+
+    if (sh->disable_deblocking_filter_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_DEBLOCKING_FILTER_DISABLED;
+
+    if (sh->slice_loop_filter_across_slices_enabled_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED;
+
+    if (sh->dependent_slice_segment_flag)
+        slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT;
+
+    if (sh->slice_type != HEVC_SLICE_I) {
+        rpl = &h->cur_frame->refPicList[0];
+        for (i = 0; i < rpl->nb_refs; i++)
+            slice_params->ref_idx_l0[i] = get_ref_pic_index(h, rpl->ref[i], decode_params);
+    }
+
+    if (sh->slice_type == HEVC_SLICE_B) {
+        rpl = &h->cur_frame->refPicList[1];
+        for (i = 0; i < rpl->nb_refs; i++)
+            slice_params->ref_idx_l1[i] = get_ref_pic_index(h, rpl->ref[i], decode_params);
+    }
+
+    fill_pred_weight_table(&slice_params->pred_weight_table, h);
+
+    if (!max_entry_point_offsets)
+        return 0;
+
+    if (controls->allocated_entry_point_offsets < controls->num_entry_point_offsets + sh->num_entry_point_offsets) {
+        void *entry_point_offsets = controls->entry_point_offsets;
+        offsets = controls->allocated_entry_point_offsets == 0 ? 128 : controls->allocated_entry_point_offsets * 2;
+        while (controls->num_entry_point_offsets + sh->num_entry_point_offsets > offsets)
+            offsets *= 2;
+        entry_point_offsets = av_realloc_array(entry_point_offsets, offsets, sizeof(*controls->entry_point_offsets));
+        if (!entry_point_offsets)
+            return AVERROR(ENOMEM);
+        controls->entry_point_offsets = entry_point_offsets;
+        controls->allocated_entry_point_offsets = offsets;
+    }
+
+    for (i = 0, offsets = controls->num_entry_point_offsets; i < sh->num_entry_point_offsets; i++)
+        controls->entry_point_offsets[offsets + i] = sh->entry_point_offset[i];
+    controls->num_entry_point_offsets += sh->num_entry_point_offsets;
+
+    return 0;
+}
+
+static void fill_sps(struct v4l2_ctrl_hevc_sps *ctrl, const HEVCContext *h)
+{
+    const HEVCPPS *pps = h->pps;
+    const HEVCSPS *sps = pps->sps;
+
+    /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */
+    *ctrl = (struct v4l2_ctrl_hevc_sps) {
+        .video_parameter_set_id = sps->vps_id,
+        .seq_parameter_set_id = pps->sps_id,
+        .pic_width_in_luma_samples = sps->width,
+        .pic_height_in_luma_samples = sps->height,
+        .bit_depth_luma_minus8 = sps->bit_depth - 8,
+        .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
+        .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4,
+        .sps_max_dec_pic_buffering_minus1 =
+            sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering - 1,
+        .sps_max_num_reorder_pics =
+            sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics,
+        .sps_max_latency_increase_plus1 =
+            sps->temporal_layer[sps->max_sub_layers - 1].max_latency_increase + 1,
+        .log2_min_luma_coding_block_size_minus3 = sps->log2_min_cb_size - 3,
+        .log2_diff_max_min_luma_coding_block_size =
+            sps->log2_diff_max_min_coding_block_size,
+        .log2_min_luma_transform_block_size_minus2 = sps->log2_min_tb_size - 2,
+        .log2_diff_max_min_luma_transform_block_size =
+            sps->log2_max_trafo_size - sps->log2_min_tb_size,
+        .max_transform_hierarchy_depth_inter = sps->max_transform_hierarchy_depth_inter,
+        .max_transform_hierarchy_depth_intra = sps->max_transform_hierarchy_depth_intra,
+        .pcm_sample_bit_depth_luma_minus1 = sps->pcm.bit_depth - 1,
+        .pcm_sample_bit_depth_chroma_minus1 = sps->pcm.bit_depth_chroma - 1,
+        .log2_min_pcm_luma_coding_block_size_minus3 = sps->pcm.log2_min_pcm_cb_size - 3,
+        .log2_diff_max_min_pcm_luma_coding_block_size =
+            sps->pcm.log2_max_pcm_cb_size - sps->pcm.log2_min_pcm_cb_size,
+        .num_short_term_ref_pic_sets = sps->nb_st_rps,
+        .num_long_term_ref_pics_sps = sps->num_long_term_ref_pics_sps,
+        .chroma_format_idc = sps->chroma_format_idc,
+        .sps_max_sub_layers_minus1 = sps->max_sub_layers - 1,
+    };
+
+    if (sps->separate_colour_plane)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_SEPARATE_COLOUR_PLANE;
+
+    if (sps->scaling_list_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED;
+
+    if (sps->amp_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_AMP_ENABLED;
+
+    if (sps->sao_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET;
+
+    if (sps->pcm_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_ENABLED;
+
+    if (sps->pcm_loop_filter_disabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED;
+
+    if (sps->long_term_ref_pics_present)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT;
+
+    if (sps->temporal_mvp_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED;
+
+    if (sps->strong_intra_smoothing_enabled)
+        ctrl->flags |= V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED;
+}
+
+static int v4l2_request_hevc_start_frame(AVCodecContext *avctx,
+                                         av_unused const uint8_t *buffer,
+                                         av_unused uint32_t size)
+{
+    const HEVCContext *h = avctx->priv_data;
+    const HEVCPPS *pps = h->pps;
+    const HEVCSPS *sps = pps->sps;
+    V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data;
+    V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private;
+    const SliceHeader *sh = &h->sh;
+    int ret;
+
+    ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_frame->f);
+    if (ret)
+        return ret;
+
+    fill_sps(&controls->sps, h);
+    fill_decode_params(&controls->decode_params, h);
+
+    if (ctx->has_scaling_matrix) {
+        const ScalingList *sl = pps->scaling_list_data_present_flag ?
+                                &pps->scaling_list :
+                                sps->scaling_list_enabled ?
+                                &sps->scaling_list : NULL;
+        if (sl) {
+            for (int i = 0; i < 6; i++) {
+                for (int j = 0; j < 16; j++)
+                    controls->scaling_matrix.scaling_list_4x4[i][j] = sl->sl[0][i][j];
+                for (int j = 0; j < 64; j++) {
+                    controls->scaling_matrix.scaling_list_8x8[i][j]   = sl->sl[1][i][j];
+                    controls->scaling_matrix.scaling_list_16x16[i][j] = sl->sl[2][i][j];
+                    if (i < 2)
+                        controls->scaling_matrix.scaling_list_32x32[i][j] = sl->sl[3][i * 3][j];
+                }
+                controls->scaling_matrix.scaling_list_dc_coef_16x16[i] = sl->sl_dc[0][i];
+                if (i < 2)
+                    controls->scaling_matrix.scaling_list_dc_coef_32x32[i] = sl->sl_dc[1][i * 3];
+            }
+        }
+    }
+
+    /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */
+    controls->pps = (struct v4l2_ctrl_hevc_pps) {
+        .pic_parameter_set_id = sh->pps_id,
+        .num_extra_slice_header_bits = pps->num_extra_slice_header_bits,
+        .num_ref_idx_l0_default_active_minus1 = pps->num_ref_idx_l0_default_active - 1,
+        .num_ref_idx_l1_default_active_minus1 = pps->num_ref_idx_l1_default_active - 1,
+        .init_qp_minus26 = pps->pic_init_qp_minus26,
+        .diff_cu_qp_delta_depth = pps->diff_cu_qp_delta_depth,
+        .pps_cb_qp_offset = pps->cb_qp_offset,
+        .pps_cr_qp_offset = pps->cr_qp_offset,
+        .pps_beta_offset_div2 = pps->beta_offset / 2,
+        .pps_tc_offset_div2 = pps->tc_offset / 2,
+        .log2_parallel_merge_level_minus2 = pps->log2_parallel_merge_level - 2,
+    };
+
+    if (pps->dependent_slice_segments_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED;
+
+    if (pps->output_flag_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT;
+
+    if (pps->sign_data_hiding_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED;
+
+    if (pps->cabac_init_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT;
+
+    if (pps->constrained_intra_pred_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CONSTRAINED_INTRA_PRED;
+
+    if (pps->transform_skip_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TRANSFORM_SKIP_ENABLED;
+
+    if (pps->cu_qp_delta_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED;
+
+    if (pps->pic_slice_level_chroma_qp_offsets_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT;
+
+    if (pps->weighted_pred_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_WEIGHTED_PRED;
+
+    if (pps->weighted_bipred_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_WEIGHTED_BIPRED;
+
+    if (pps->transquant_bypass_enable_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TRANSQUANT_BYPASS_ENABLED;
+
+    if (pps->tiles_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_TILES_ENABLED;
+
+    if (pps->entropy_coding_sync_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED;
+
+    if (pps->loop_filter_across_tiles_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED;
+
+    if (pps->seq_loop_filter_across_slices_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED;
+
+    if (pps->deblocking_filter_override_enabled_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_OVERRIDE_ENABLED;
+
+    if (pps->disable_dbf)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER;
+
+    if (pps->lists_modification_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT;
+
+    if (pps->slice_header_extension_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT;
+
+    if (pps->deblocking_filter_control_present_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT;
+
+    if (pps->uniform_spacing_flag)
+        controls->pps.flags |= V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING;
+
+    if (pps->tiles_enabled_flag) {
+        controls->pps.num_tile_columns_minus1 = pps->num_tile_columns - 1;
+        controls->pps.num_tile_rows_minus1 = pps->num_tile_rows - 1;
+
+        for (int i = 0; i < pps->num_tile_columns; i++)
+            controls->pps.column_width_minus1[i] = pps->column_width[i] - 1;
+
+        for (int i = 0; i < pps->num_tile_rows; i++)
+            controls->pps.row_height_minus1[i] = pps->row_height[i] - 1;
+    }
+
+    controls->first_slice = true;
+    controls->frame_slice_params = &controls->slice_params;
+    controls->allocated_slice_params = 0;
+    controls->num_slice_params = 0;
+    controls->allocated_entry_point_offsets = 0;
+    controls->num_entry_point_offsets = 0;
+
+    return 0;
+}
+
+static int v4l2_request_hevc_queue_decode(AVCodecContext *avctx, bool last_slice)
+{
+    const HEVCContext *h = avctx->priv_data;
+    V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data;
+    V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private;
+    int count = 0;
+
+    struct v4l2_ext_control control[V4L2_HEVC_CONTROLS_MAX] = {};
+
+    control[count++] = (struct v4l2_ext_control) {
+        .id = V4L2_CID_STATELESS_HEVC_SPS,
+        .ptr = &controls->sps,
+        .size = sizeof(controls->sps),
+    };
+
+    control[count++] = (struct v4l2_ext_control) {
+        .id = V4L2_CID_STATELESS_HEVC_PPS,
+        .ptr = &controls->pps,
+        .size = sizeof(controls->pps),
+    };
+
+    control[count++] = (struct v4l2_ext_control) {
+        .id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS,
+        .ptr = &controls->decode_params,
+        .size = sizeof(controls->decode_params),
+    };
+
+    if (ctx->has_scaling_matrix) {
+        control[count++] = (struct v4l2_ext_control) {
+            .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX,
+            .ptr = &controls->scaling_matrix,
+            .size = sizeof(controls->scaling_matrix),
+        };
+    }
+
+    if (ctx->max_slice_params && controls->num_slice_params) {
+        control[count++] = (struct v4l2_ext_control) {
+            .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS,
+            .ptr = controls->frame_slice_params,
+            .size = sizeof(*controls->frame_slice_params) *
+                    FFMIN(controls->num_slice_params, ctx->max_slice_params),
+        };
+    }
+
+    if (ctx->max_entry_point_offsets && controls->num_entry_point_offsets) {
+        control[count++] = (struct v4l2_ext_control) {
+            .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS,
+            .ptr = controls->entry_point_offsets,
+            .size = sizeof(*controls->entry_point_offsets) *
+                    FFMIN(controls->num_entry_point_offsets,
+                          ctx->max_entry_point_offsets),
+        };
+    }
+
+    if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED)
+        return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count,
+                                            controls->first_slice, last_slice);
+
+    return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, count);
+}
+
+static int v4l2_request_hevc_decode_slice(AVCodecContext *avctx,
+                                          const uint8_t *buffer, uint32_t size)
+{
+    const HEVCContext *h = avctx->priv_data;
+    V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data;
+    V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private;
+    const SliceHeader *sh = &h->sh;
+    int ret, slice = controls->num_slice_params;
+    uint32_t extra_size = 0;
+
+    if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED &&
+        (slice >= ctx->max_slice_params || (ctx->max_entry_point_offsets &&
+         (controls->num_entry_point_offsets + sh->num_entry_point_offsets > ctx->max_entry_point_offsets)))) {
+        ret = v4l2_request_hevc_queue_decode(avctx, false);
+        if (ret)
+            return ret;
+
+        ff_v4l2_request_reset_picture(avctx, &controls->pic);
+        slice = controls->num_slice_params = 0;
+        controls->num_entry_point_offsets = 0;
+        controls->first_slice = false;
+    }
+
+    if (ctx->start_code == V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) {
+        ret = ff_v4l2_request_append_output(avctx, &controls->pic,
+                                            nalu_slice_start_code, 3);
+        if (ret)
+            return ret;
+        extra_size = 3;
+    }
+
+    if (ctx->max_slice_params) {
+        if (slice && controls->allocated_slice_params < slice + 1) {
+            void *slice_params = controls->allocated_slice_params == 0 ? NULL : controls->frame_slice_params;
+            int slices = controls->allocated_slice_params == 0 ? 8 : controls->allocated_slice_params * 2;
+            slice_params = av_realloc_array(slice_params, slices, sizeof(*controls->frame_slice_params));
+            if (!slice_params)
+                return AVERROR(ENOMEM);
+            if (controls->allocated_slice_params == 0)
+                memcpy(slice_params, controls->frame_slice_params, sizeof(*controls->frame_slice_params));
+            controls->frame_slice_params = slice_params;
+            controls->allocated_slice_params = slices;
+        }
+
+        ret = fill_slice_params(controls, slice, !!ctx->max_entry_point_offsets, h);
+        if (ret)
+            return ret;
+    }
+
+    ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size);
+    if (ret)
+        return ret;
+
+    if (ctx->max_slice_params)
+        controls->frame_slice_params[slice].bit_size = (size + extra_size) * 8;
+
+    controls->num_slice_params++;
+    return 0;
+}
+
+static int v4l2_request_hevc_end_frame(AVCodecContext *avctx)
+{
+    return v4l2_request_hevc_queue_decode(avctx, true);
+}
+
+static void v4l2_request_hevc_free_frame_priv(FFRefStructOpaque hwctx, void *data)
+{
+    V4L2RequestControlsHEVC *controls = data;
+
+    if (controls->allocated_slice_params)
+        av_freep(&controls->frame_slice_params);
+
+    av_freep(&controls->entry_point_offsets);
+}
+
+static int v4l2_request_hevc_post_probe(AVCodecContext *avctx)
+{
+    V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data;
+    int ret;
+
+    struct v4l2_ext_control control[] = {
+        { .id = V4L2_CID_STATELESS_HEVC_DECODE_MODE, },
+        { .id = V4L2_CID_STATELESS_HEVC_START_CODE, },
+    };
+    struct v4l2_query_ext_ctrl scaling_matrix = {
+        .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX,
+    };
+    struct v4l2_query_ext_ctrl entry_point_offsets = {
+        .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS,
+    };
+    struct v4l2_query_ext_ctrl slice_params = {
+        .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS,
+    };
+
+    ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx,
+                                        V4L2_CID_STATELESS_HEVC_DECODE_MODE);
+    if (ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED &&
+        ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED) {
+        av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n",
+               ctx->decode_mode);
+        return AVERROR(EINVAL);
+    }
+
+    ctx->start_code = ff_v4l2_request_query_control_default_value(avctx,
+                                        V4L2_CID_STATELESS_HEVC_START_CODE);
+    if (ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_NONE &&
+        ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) {
+        av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n",
+               ctx->start_code);
+        return AVERROR(EINVAL);
+    }
+
+    // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_PROFILE control
+    // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_LEVEL control
+
+    ret = ff_v4l2_request_query_control(avctx, &scaling_matrix);
+    if (!ret)
+        ctx->has_scaling_matrix = true;
+    else
+        ctx->has_scaling_matrix = false;
+
+    ret = ff_v4l2_request_query_control(avctx, &entry_point_offsets);
+    if (!ret)
+        ctx->max_entry_point_offsets = FFMAX(entry_point_offsets.dims[0], 1);
+    else
+        ctx->max_entry_point_offsets = 0;
+
+    ret = ff_v4l2_request_query_control(avctx, &slice_params);
+    if (!ret)
+        ctx->max_slice_params = FFMAX(slice_params.dims[0], 1);
+    else
+        ctx->max_slice_params = 0;
+
+    av_log(ctx, AV_LOG_VERBOSE, "%s-based decoder with SLICE_PARAMS=%u, "
+           "ENTRY_POINT_OFFSETS=%u and SCALING_MATRIX=%d controls\n",
+          ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED ? "slice" : "frame",
+          ctx->max_slice_params, ctx->max_entry_point_offsets, ctx->has_scaling_matrix);
+
+    control[0].value = ctx->decode_mode;
+    control[1].value = ctx->start_code;
+
+    return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control));
+}
+
+static int v4l2_request_hevc_init(AVCodecContext *avctx)
+{
+    V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data;
+    const HEVCContext *h = avctx->priv_data;
+    struct v4l2_ctrl_hevc_sps sps;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_HEVC_SPS,
+            .ptr = &sps,
+            .size = sizeof(sps),
+        },
+    };
+
+    fill_sps(&sps, h);
+
+    ctx->base.post_probe = v4l2_request_hevc_post_probe;
+    return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_HEVC_SLICE,
+                                4 * 1024 * 1024,
+                                control, FF_ARRAY_ELEMS(control));
+}
+
+const FFHWAccel ff_hevc_v4l2request_hwaccel = {
+    .p.name             = "hevc_v4l2request",
+    .p.type             = AVMEDIA_TYPE_VIDEO,
+    .p.id               = AV_CODEC_ID_HEVC,
+    .p.pix_fmt          = AV_PIX_FMT_DRM_PRIME,
+    .start_frame        = v4l2_request_hevc_start_frame,
+    .decode_slice       = v4l2_request_hevc_decode_slice,
+    .end_frame          = v4l2_request_hevc_end_frame,
+    .flush              = ff_v4l2_request_flush,
+    .free_frame_priv    = v4l2_request_hevc_free_frame_priv,
+    .frame_priv_data_size = sizeof(V4L2RequestControlsHEVC),
+    .init               = v4l2_request_hevc_init,
+    .uninit             = ff_v4l2_request_uninit,
+    .priv_data_size     = sizeof(V4L2RequestContextHEVC),
+    .frame_params       = ff_v4l2_request_frame_params,
+};

From 63716a8632a02eb3c9d7715a5dd0bf99d4296d48 Mon Sep 17 00:00:00 2001
From: Boris Brezillon <boris.brezillon@collabora.com>
Date: Wed, 22 May 2019 14:46:58 +0200
Subject: [PATCH 09/11] avcodec: Add V4L2 Request API vp8 hwaccel

Add a V4L2 Request API hwaccel for VP8.

Co-developed-by: Ezequiel Garcia <ezequiel@collabora.com>
Signed-off-by: Ezequiel Garcia <ezequiel@collabora.com>
Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Co-developed-by: Jonas Karlman <jonas@kwiboo.se>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                     |   3 +
 libavcodec/Makefile           |   1 +
 libavcodec/hwaccels.h         |   1 +
 libavcodec/v4l2_request_vp8.c | 235 ++++++++++++++++++++++++++++++++++
 libavcodec/vp8.c              |   6 +
 5 files changed, 246 insertions(+)
 create mode 100644 libavcodec/v4l2_request_vp8.c

diff --git a/configure b/configure
index 7a2901d960..3b1288eb98 100755
--- a/configure
+++ b/configure
@@ -3271,6 +3271,8 @@ vc1_vdpau_hwaccel_deps="vdpau"
 vc1_vdpau_hwaccel_select="vc1_decoder"
 vp8_nvdec_hwaccel_deps="nvdec"
 vp8_nvdec_hwaccel_select="vp8_decoder"
+vp8_v4l2request_hwaccel_deps="v4l2_request vp8_v4l2_request"
+vp8_v4l2request_hwaccel_select="vp8_decoder"
 vp8_vaapi_hwaccel_deps="vaapi"
 vp8_vaapi_hwaccel_select="vp8_decoder"
 vp9_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_VP9"
@@ -7213,6 +7215,7 @@ if enabled v4l2_request; then
     check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS"
     check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE"
     check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
+    check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
     check_pkg_config libudev libudev libudev.h udev_new
     check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index fa78523b76..3fa9822594 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1057,6 +1057,7 @@ OBJS-$(CONFIG_VC1_QSV_HWACCEL)            += qsvdec.o
 OBJS-$(CONFIG_VC1_VAAPI_HWACCEL)          += vaapi_vc1.o
 OBJS-$(CONFIG_VC1_VDPAU_HWACCEL)          += vdpau_vc1.o
 OBJS-$(CONFIG_VP8_NVDEC_HWACCEL)          += nvdec_vp8.o
+OBJS-$(CONFIG_VP8_V4L2REQUEST_HWACCEL)    += v4l2_request_vp8.o
 OBJS-$(CONFIG_VP8_VAAPI_HWACCEL)          += vaapi_vp8.o
 OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL)        += dxva2_vp9.o
 OBJS-$(CONFIG_VP9_DXVA2_HWACCEL)          += dxva2_vp9.o
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index ad145fc07c..cb2cdb3893 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -76,6 +76,7 @@ extern const struct FFHWAccel ff_vc1_nvdec_hwaccel;
 extern const struct FFHWAccel ff_vc1_vaapi_hwaccel;
 extern const struct FFHWAccel ff_vc1_vdpau_hwaccel;
 extern const struct FFHWAccel ff_vp8_nvdec_hwaccel;
+extern const struct FFHWAccel ff_vp8_v4l2request_hwaccel;
 extern const struct FFHWAccel ff_vp8_vaapi_hwaccel;
 extern const struct FFHWAccel ff_vp9_d3d11va_hwaccel;
 extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel;
diff --git a/libavcodec/v4l2_request_vp8.c b/libavcodec/v4l2_request_vp8.c
new file mode 100644
index 0000000000..551b6dc75f
--- /dev/null
+++ b/libavcodec/v4l2_request_vp8.c
@@ -0,0 +1,235 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include "hwaccel_internal.h"
+#include "hwconfig.h"
+#include "v4l2_request.h"
+#include "vp8.h"
+
+typedef struct V4L2RequestControlsVP8 {
+    V4L2RequestPictureContext pic;
+    struct v4l2_ctrl_vp8_frame frame;
+} V4L2RequestControlsVP8;
+
+static int v4l2_request_vp8_start_frame(AVCodecContext *avctx,
+                                        const uint8_t *buffer,
+                                        av_unused uint32_t size)
+{
+    const VP8Context *s = avctx->priv_data;
+    V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private;
+    struct v4l2_ctrl_vp8_frame *ctrl = &controls->frame;
+    unsigned int header_size = 3 + 7 * s->keyframe;
+    const uint8_t *data = buffer + header_size;
+    int ret, i, j, k;
+
+    ret = ff_v4l2_request_start_frame(avctx, &controls->pic,
+                                      s->framep[VP8_FRAME_CURRENT]->tf.f);
+    if (ret)
+        return ret;
+
+    *ctrl = (struct v4l2_ctrl_vp8_frame) {
+        .lf = {
+            .sharpness_level = s->filter.sharpness,
+            .level = s->filter.level,
+        },
+
+        .quant = {
+            .y_ac_qi = s->quant.yac_qi,
+            .y_dc_delta = s->quant.ydc_delta,
+            .y2_dc_delta = s->quant.y2dc_delta,
+            .y2_ac_delta = s->quant.y2ac_delta,
+            .uv_dc_delta = s->quant.uvdc_delta,
+            .uv_ac_delta = s->quant.uvac_delta,
+        },
+
+        .coder_state = {
+            .range = s->coder_state_at_header_end.range,
+            .value = s->coder_state_at_header_end.value,
+            .bit_count = s->coder_state_at_header_end.bit_count,
+        },
+
+        .width = avctx->width,
+        .height = avctx->height,
+
+        .horizontal_scale = 0, /* scale not supported by FFmpeg */
+        .vertical_scale = 0, /* scale not supported by FFmpeg */
+
+        .version = s->profile & 0x3,
+        .prob_skip_false = s->prob->mbskip,
+        .prob_intra = s->prob->intra,
+        .prob_last = s->prob->last,
+        .prob_gf = s->prob->golden,
+        .num_dct_parts = s->num_coeff_partitions,
+
+        .first_part_size = s->header_partition_size,
+        .first_part_header_bits = (8 * (s->coder_state_at_header_end.input - data) -
+                                   s->coder_state_at_header_end.bit_count - 8),
+    };
+
+    for (i = 0; i < 4; i++) {
+        ctrl->segment.quant_update[i] = s->segmentation.base_quant[i];
+        ctrl->segment.lf_update[i] = s->segmentation.filter_level[i];
+    }
+
+    for (i = 0; i < 3; i++)
+        ctrl->segment.segment_probs[i] = s->prob->segmentid[i];
+
+    if (s->segmentation.enabled)
+        ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_ENABLED;
+
+    if (s->segmentation.update_map)
+        ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_MAP;
+
+    if (s->segmentation.update_feature_data)
+        ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_FEATURE_DATA;
+
+    if (!s->segmentation.absolute_vals)
+        ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_DELTA_VALUE_MODE;
+
+    for (i = 0; i < 4; i++) {
+        ctrl->lf.ref_frm_delta[i] = s->lf_delta.ref[i];
+        ctrl->lf.mb_mode_delta[i] = s->lf_delta.mode[i + MODE_I4x4];
+    }
+
+    if (s->lf_delta.enabled)
+        ctrl->lf.flags |= V4L2_VP8_LF_ADJ_ENABLE;
+
+    if (s->lf_delta.update)
+        ctrl->lf.flags |= V4L2_VP8_LF_DELTA_UPDATE;
+
+    if (s->filter.simple)
+        ctrl->lf.flags |= V4L2_VP8_LF_FILTER_TYPE_SIMPLE;
+
+    if (s->keyframe) {
+        static const uint8_t keyframe_y_mode_probs[4] = {
+            145, 156, 163, 128
+        };
+        static const uint8_t keyframe_uv_mode_probs[3] = {
+            142, 114, 183
+        };
+
+        memcpy(ctrl->entropy.y_mode_probs, keyframe_y_mode_probs, 4);
+        memcpy(ctrl->entropy.uv_mode_probs, keyframe_uv_mode_probs, 3);
+    } else {
+        for (i = 0; i < 4; i++)
+            ctrl->entropy.y_mode_probs[i] = s->prob->pred16x16[i];
+        for (i = 0; i < 3; i++)
+            ctrl->entropy.uv_mode_probs[i] = s->prob->pred8x8c[i];
+    }
+    for (i = 0; i < 2; i++)
+        for (j = 0; j < 19; j++)
+            ctrl->entropy.mv_probs[i][j] = s->prob->mvc[i][j];
+
+    for (i = 0; i < 4; i++) {
+        for (j = 0; j < 8; j++) {
+            static const int coeff_bands_inverse[8] = {
+                0, 1, 2, 3, 5, 6, 4, 15
+            };
+            int coeff_pos = coeff_bands_inverse[j];
+
+            for (k = 0; k < 3; k++) {
+                memcpy(ctrl->entropy.coeff_probs[i][j][k],
+                       s->prob->token[i][coeff_pos][k], 11);
+            }
+        }
+    }
+
+    for (i = 0; i < 8; i++)
+        ctrl->dct_part_sizes[i] = s->coeff_partition_size[i];
+
+    if (s->framep[VP8_FRAME_PREVIOUS])
+        ctrl->last_frame_ts =
+            ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_PREVIOUS]->tf.f);
+    if (s->framep[VP8_FRAME_GOLDEN])
+        ctrl->golden_frame_ts =
+            ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_GOLDEN]->tf.f);
+    if (s->framep[VP8_FRAME_ALTREF])
+        ctrl->alt_frame_ts =
+            ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_ALTREF]->tf.f);
+
+    if (s->keyframe)
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_KEY_FRAME;
+
+    if (s->profile & 0x4)
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_EXPERIMENTAL;
+
+    if (!s->invisible)
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_SHOW_FRAME;
+
+    if (s->mbskip_enabled)
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_MB_NO_SKIP_COEFF;
+
+    if (s->sign_bias[VP8_FRAME_GOLDEN])
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_GOLDEN;
+
+    if (s->sign_bias[VP8_FRAME_ALTREF])
+        ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_ALT;
+
+    return 0;
+}
+
+static int v4l2_request_vp8_decode_slice(AVCodecContext *avctx,
+                                         const uint8_t *buffer, uint32_t size)
+{
+    const VP8Context *s = avctx->priv_data;
+    V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private;
+
+    return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size);
+}
+
+static int v4l2_request_vp8_end_frame(AVCodecContext *avctx)
+{
+    const VP8Context *s = avctx->priv_data;
+    V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_VP8_FRAME,
+            .ptr = &controls->frame,
+            .size = sizeof(controls->frame),
+        },
+    };
+
+    return ff_v4l2_request_decode_frame(avctx, &controls->pic,
+                                        control, FF_ARRAY_ELEMS(control));
+}
+
+static int v4l2_request_vp8_init(AVCodecContext *avctx)
+{
+    return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP8_FRAME,
+                                2 * 1024 * 1024,
+                                NULL, 0);
+}
+
+const FFHWAccel ff_vp8_v4l2request_hwaccel = {
+    .p.name             = "vp8_v4l2request",
+    .p.type             = AVMEDIA_TYPE_VIDEO,
+    .p.id               = AV_CODEC_ID_VP8,
+    .p.pix_fmt          = AV_PIX_FMT_DRM_PRIME,
+    .start_frame        = v4l2_request_vp8_start_frame,
+    .decode_slice       = v4l2_request_vp8_decode_slice,
+    .end_frame          = v4l2_request_vp8_end_frame,
+    .flush              = ff_v4l2_request_flush,
+    .frame_priv_data_size = sizeof(V4L2RequestControlsVP8),
+    .init               = v4l2_request_vp8_init,
+    .uninit             = ff_v4l2_request_uninit,
+    .priv_data_size     = sizeof(V4L2RequestContext),
+    .frame_params       = ff_v4l2_request_frame_params,
+};
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index d6df018655..04b16cc2e3 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -184,6 +184,9 @@ static enum AVPixelFormat get_pixel_format(VP8Context *s)
 #endif
 #if CONFIG_VP8_NVDEC_HWACCEL
         AV_PIX_FMT_CUDA,
+#endif
+#if CONFIG_VP8_V4L2REQUEST_HWACCEL
+        AV_PIX_FMT_DRM_PRIME,
 #endif
         AV_PIX_FMT_YUV420P,
         AV_PIX_FMT_NONE,
@@ -2980,6 +2983,9 @@ const FFCodec ff_vp8_decoder = {
 #endif
 #if CONFIG_VP8_NVDEC_HWACCEL
                                HWACCEL_NVDEC(vp8),
+#endif
+#if CONFIG_VP8_V4L2REQUEST_HWACCEL
+                               HWACCEL_V4L2REQUEST(vp8),
 #endif
                                NULL
                            },

From 96c22879bbbfcbd57cd73a165ab6b1fb49663481 Mon Sep 17 00:00:00 2001
From: Boris Brezillon <boris.brezillon@collabora.com>
Date: Thu, 12 Dec 2019 16:13:55 +0100
Subject: [PATCH 10/11] avcodec: Add V4L2 Request API vp9 hwaccel

FIXME: split patch

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Co-developed-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Signed-off-by: Jernej Skrabec <jernej.skrabec@gmail.com>
Co-developed-by: Jonas Karlman <jonas@kwiboo.se>
Signed-off-by: Jonas Karlman <jonas@kwiboo.se>
---
 configure                     |   3 +
 libavcodec/Makefile           |   1 +
 libavcodec/hwaccels.h         |   1 +
 libavcodec/v4l2_request_vp9.c | 288 ++++++++++++++++++++++++++++++++++
 libavcodec/vp9.c              | 187 +++++++++++++++-------
 libavcodec/vp9dec.h           |   4 +
 libavcodec/vp9shared.h        |   1 +
 7 files changed, 431 insertions(+), 54 deletions(-)
 create mode 100644 libavcodec/v4l2_request_vp9.c

diff --git a/configure b/configure
index 3b1288eb98..2904b67ef7 100755
--- a/configure
+++ b/configure
@@ -3285,6 +3285,8 @@ vp9_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_VP9"
 vp9_dxva2_hwaccel_select="vp9_decoder"
 vp9_nvdec_hwaccel_deps="nvdec"
 vp9_nvdec_hwaccel_select="vp9_decoder"
+vp9_v4l2request_hwaccel_deps="v4l2_request vp9_v4l2_request"
+vp9_v4l2request_hwaccel_select="vp9_decoder"
 vp9_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferVP9_bit_depth"
 vp9_vaapi_hwaccel_select="vp9_decoder"
 vp9_vdpau_hwaccel_deps="vdpau VdpPictureInfoVP9"
@@ -7216,6 +7218,7 @@ if enabled v4l2_request; then
     check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE"
     check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF"
     check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME"
+    check_cc vp9_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP9_FRAME"
     check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
     check_pkg_config libudev libudev libudev.h udev_new
     check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 3fa9822594..95c77d7bec 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1063,6 +1063,7 @@ OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL)        += dxva2_vp9.o
 OBJS-$(CONFIG_VP9_DXVA2_HWACCEL)          += dxva2_vp9.o
 OBJS-$(CONFIG_VP9_D3D12VA_HWACCEL)        += dxva2_vp9.o d3d12va_vp9.o
 OBJS-$(CONFIG_VP9_NVDEC_HWACCEL)          += nvdec_vp9.o
+OBJS-$(CONFIG_VP9_V4L2REQUEST_HWACCEL)    += v4l2_request_vp9.o
 OBJS-$(CONFIG_VP9_VAAPI_HWACCEL)          += vaapi_vp9.o
 OBJS-$(CONFIG_VP9_VDPAU_HWACCEL)          += vdpau_vp9.o
 OBJS-$(CONFIG_VP9_VIDEOTOOLBOX_HWACCEL)   += videotoolbox_vp9.o
diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h
index cb2cdb3893..8571cc7e8b 100644
--- a/libavcodec/hwaccels.h
+++ b/libavcodec/hwaccels.h
@@ -83,6 +83,7 @@ extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel;
 extern const struct FFHWAccel ff_vp9_d3d12va_hwaccel;
 extern const struct FFHWAccel ff_vp9_dxva2_hwaccel;
 extern const struct FFHWAccel ff_vp9_nvdec_hwaccel;
+extern const struct FFHWAccel ff_vp9_v4l2request_hwaccel;
 extern const struct FFHWAccel ff_vp9_vaapi_hwaccel;
 extern const struct FFHWAccel ff_vp9_vdpau_hwaccel;
 extern const struct FFHWAccel ff_vp9_videotoolbox_hwaccel;
diff --git a/libavcodec/v4l2_request_vp9.c b/libavcodec/v4l2_request_vp9.c
new file mode 100644
index 0000000000..41ea4e37f5
--- /dev/null
+++ b/libavcodec/v4l2_request_vp9.c
@@ -0,0 +1,288 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include "hwaccel_internal.h"
+#include "hwconfig.h"
+#include "v4l2_request.h"
+#include "vp9dec.h"
+
+typedef struct V4L2RequestControlsVP9 {
+    V4L2RequestPictureContext pic;
+    struct v4l2_ctrl_vp9_frame frame;
+    struct v4l2_ctrl_vp9_compressed_hdr chp;
+} V4L2RequestControlsVP9;
+
+static void v4l2_request_vp9_set_frame_ctx(AVCodecContext *avctx)
+{
+    VP9Context *s = avctx->priv_data;
+    const VP9Frame *f = &s->s.frames[CUR_FRAME];
+    V4L2RequestControlsVP9 *controls = f->hwaccel_picture_private;
+    struct v4l2_ctrl_vp9_compressed_hdr *chp = &controls->chp;
+
+    memset(chp, 0, sizeof(&chp));
+
+    chp->tx_mode = s->s.h.txfmmode;
+    memcpy(chp->tx8, s->prob_raw.p.tx8p, sizeof(s->prob_raw.p.tx8p));
+    memcpy(chp->tx16, s->prob_raw.p.tx16p, sizeof(s->prob_raw.p.tx16p));
+    memcpy(chp->tx32, s->prob_raw.p.tx32p, sizeof(s->prob_raw.p.tx32p));
+    for (unsigned i = 0; i < 4; i++) {
+        for (unsigned j = 0; j < 2; j++) {
+            for (unsigned k = 0; k < 2; k++) {
+                for (unsigned l = 0; l < 6; l++) {
+                    for (unsigned m = 0; m < 6; m++) {
+                        memcpy(chp->coef[i][j][k][l][m], s->prob_raw.coef[i][j][k][l][m], sizeof(chp->coef[0][0][0][0][0]));
+                    }
+                }
+            }
+        }
+    }
+    memcpy(chp->skip, s->prob_raw.p.skip, sizeof(s->prob_raw.p.skip));
+    memcpy(chp->inter_mode, s->prob_raw.p.mv_mode, sizeof(s->prob_raw.p.mv_mode));
+    memcpy(chp->interp_filter, s->prob_raw.p.filter, sizeof(s->prob_raw.p.filter));
+    memcpy(chp->is_inter, s->prob_raw.p.intra, sizeof(s->prob_raw.p.intra));
+    memcpy(chp->comp_mode, s->prob_raw.p.comp, sizeof(s->prob_raw.p.comp));
+    memcpy(chp->single_ref, s->prob_raw.p.single_ref, sizeof(s->prob_raw.p.single_ref));
+    memcpy(chp->comp_ref, s->prob_raw.p.comp_ref, sizeof(s->prob_raw.p.comp_ref));
+    memcpy(chp->y_mode, s->prob_raw.p.y_mode, sizeof(s->prob_raw.p.y_mode));
+    for (unsigned i = 0; i < 10; i++)
+        memcpy(chp->uv_mode[i], s->prob.p.uv_mode[i], sizeof(s->prob.p.uv_mode[0]));
+    for (unsigned i = 0; i < 4; i++)
+        memcpy(chp->partition[i * 4], s->prob_raw.p.partition[i], sizeof(s->prob_raw.p.partition[0]));
+    memcpy(chp->mv.joint, s->prob_raw.p.mv_joint, sizeof(s->prob_raw.p.mv_joint));
+    for (unsigned i = 0; i < 2; i++) {
+         chp->mv.sign[i] = s->prob_raw.p.mv_comp[i].sign;
+         memcpy(chp->mv.classes[i], s->prob_raw.p.mv_comp[i].classes, sizeof(s->prob_raw.p.mv_comp[0].classes));
+         chp->mv.class0_bit[i] = s->prob_raw.p.mv_comp[i].class0;
+         memcpy(chp->mv.bits[i], s->prob_raw.p.mv_comp[i].bits, sizeof(s->prob_raw.p.mv_comp[0].bits));
+         memcpy(chp->mv.class0_fr[i], s->prob_raw.p.mv_comp[i].class0_fp, sizeof(s->prob_raw.p.mv_comp[0].class0_fp));
+         memcpy(chp->mv.fr[i], s->prob_raw.p.mv_comp[i].fp, sizeof(s->prob_raw.p.mv_comp[0].fp));
+         chp->mv.class0_hp[i] = s->prob_raw.p.mv_comp[i].class0_hp;
+         chp->mv.hp[i] = s->prob_raw.p.mv_comp[i].hp;
+    }
+}
+
+static void fill_frame(struct v4l2_ctrl_vp9_frame *dec_params, AVCodecContext *avctx)
+{
+    const VP9Context *s = avctx->priv_data;
+    AVFrame *ref;
+
+    memset(dec_params, 0, sizeof(*dec_params));
+
+    if (s->s.h.keyframe)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_KEY_FRAME;
+    if (!s->s.h.invisible)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_SHOW_FRAME;
+    if (s->s.h.errorres)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT;
+    if (s->s.h.intraonly)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_INTRA_ONLY;
+    if (!s->s.h.keyframe && s->s.h.highprecisionmvs)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_ALLOW_HIGH_PREC_MV;
+    if (s->s.h.refreshctx)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX;
+    if (s->s.h.parallelmode)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE;
+    if (s->ss_h)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_X_SUBSAMPLING;
+    if (s->ss_v)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_Y_SUBSAMPLING;
+    if (avctx->color_range == AVCOL_RANGE_JPEG)
+        dec_params->flags |= V4L2_VP9_FRAME_FLAG_COLOR_RANGE_FULL_SWING;
+
+    dec_params->compressed_header_size = s->s.h.compressed_header_size;
+    dec_params->uncompressed_header_size = s->s.h.uncompressed_header_size;
+    dec_params->profile = s->s.h.profile;
+    dec_params->reset_frame_context = s->s.h.resetctx > 0 ? s->s.h.resetctx - 1 : 0;
+    dec_params->frame_context_idx = s->s.h.framectxid;
+    dec_params->bit_depth = s->s.h.bpp;
+
+    dec_params->interpolation_filter = s->s.h.filtermode ^ (s->s.h.filtermode <= 1);
+    dec_params->tile_cols_log2 = s->s.h.tiling.log2_tile_cols;
+    dec_params->tile_rows_log2 = s->s.h.tiling.log2_tile_rows;
+    dec_params->reference_mode = s->s.h.comppredmode;
+    dec_params->frame_width_minus_1 = s->w - 1;
+    dec_params->frame_height_minus_1 = s->h - 1;
+    //dec_params->render_width_minus_1 = avctx->width - 1;
+    //dec_params->render_height_minus_1 = avctx->height - 1;
+
+    ref = s->s.refs[s->s.h.refidx[0]].f;
+    if (ref && ref->buf[0])
+        dec_params->last_frame_ts = ff_v4l2_request_get_capture_timestamp(ref);
+    ref = s->s.refs[s->s.h.refidx[1]].f;
+    if (ref && ref->buf[0])
+        dec_params->golden_frame_ts = ff_v4l2_request_get_capture_timestamp(ref);
+    ref = s->s.refs[s->s.h.refidx[2]].f;
+    if (ref && ref->buf[0])
+        dec_params->alt_frame_ts = ff_v4l2_request_get_capture_timestamp(ref);
+
+    if (s->s.h.signbias[0])
+        dec_params->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_LAST;
+    if (s->s.h.signbias[1])
+        dec_params->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_GOLDEN;
+    if (s->s.h.signbias[2])
+        dec_params->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_ALT;
+
+    if (s->s.h.lf_delta.enabled)
+        dec_params->lf.flags |= V4L2_VP9_LOOP_FILTER_FLAG_DELTA_ENABLED;
+    if (s->s.h.lf_delta.updated)
+        dec_params->lf.flags |= V4L2_VP9_LOOP_FILTER_FLAG_DELTA_UPDATE;
+
+    dec_params->lf.level = s->s.h.filter.level;
+    dec_params->lf.sharpness = s->s.h.filter.sharpness;
+    for (unsigned i = 0; i < 4; i++)
+        dec_params->lf.ref_deltas[i] = s->s.h.lf_delta.ref[i];
+    for (unsigned i = 0; i < 2; i++)
+        dec_params->lf.mode_deltas[i] = s->s.h.lf_delta.mode[i];
+
+    dec_params->quant.base_q_idx = s->s.h.yac_qi;
+    dec_params->quant.delta_q_y_dc = s->s.h.ydc_qdelta;
+    dec_params->quant.delta_q_uv_dc = s->s.h.uvdc_qdelta;
+    dec_params->quant.delta_q_uv_ac = s->s.h.uvac_qdelta;
+
+    if (s->s.h.segmentation.enabled)
+        dec_params->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_ENABLED;
+    if (s->s.h.segmentation.update_map)
+        dec_params->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_UPDATE_MAP;
+    if (s->s.h.segmentation.temporal)
+        dec_params->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_TEMPORAL_UPDATE;
+    if (s->s.h.segmentation.update_data)
+        dec_params->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_UPDATE_DATA;
+    if (s->s.h.segmentation.absolute_vals)
+        dec_params->seg.flags |= V4L2_VP9_SEGMENTATION_FLAG_ABS_OR_DELTA_UPDATE;
+
+    for (unsigned i = 0; i < 7; i++)
+        dec_params->seg.tree_probs[i] = s->s.h.segmentation.prob[i];
+
+    if (s->s.h.segmentation.temporal) {
+        for (unsigned i = 0; i < 3; i++)
+            dec_params->seg.pred_probs[i] = s->s.h.segmentation.pred_prob[i];
+    } else {
+        memset(dec_params->seg.pred_probs, 255, sizeof(dec_params->seg.pred_probs));
+    }
+
+    for (unsigned i = 0; i < 8; i++) {
+        if (s->s.h.segmentation.feat[i].q_enabled) {
+            dec_params->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_ALT_Q;
+            dec_params->seg.feature_data[i][V4L2_VP9_SEG_LVL_ALT_Q] = s->s.h.segmentation.feat[i].q_val;
+        }
+
+        if (s->s.h.segmentation.feat[i].lf_enabled) {
+            dec_params->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_ALT_L;
+            dec_params->seg.feature_data[i][V4L2_VP9_SEG_LVL_ALT_L] = s->s.h.segmentation.feat[i].lf_val;
+        }
+
+        if (s->s.h.segmentation.feat[i].ref_enabled) {
+            dec_params->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_REF_FRAME;
+            dec_params->seg.feature_data[i][V4L2_VP9_SEG_LVL_REF_FRAME] = s->s.h.segmentation.feat[i].ref_val;
+        }
+
+        if (s->s.h.segmentation.feat[i].skip_enabled)
+            dec_params->seg.feature_enabled[i] |= 1 << V4L2_VP9_SEG_LVL_SKIP;
+    }
+}
+
+static int v4l2_request_vp9_start_frame(AVCodecContext *avctx,
+                                        av_unused const uint8_t *buffer,
+                                        av_unused uint32_t size)
+{
+    const VP9SharedContext *h = avctx->priv_data;
+    const VP9Frame *f = &h->frames[CUR_FRAME];
+    V4L2RequestControlsVP9 *controls = f->hwaccel_picture_private;
+    int ret;
+
+    ret = ff_v4l2_request_start_frame(avctx, &controls->pic, f->tf.f);
+    if (ret)
+        return ret;
+
+    v4l2_request_vp9_set_frame_ctx(avctx);
+
+    fill_frame(&controls->frame, avctx);
+
+    return 0;
+}
+
+static int v4l2_request_vp9_decode_slice(AVCodecContext *avctx,
+                                         const uint8_t *buffer, uint32_t size)
+{
+    const VP9SharedContext *h = avctx->priv_data;
+    V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private;
+
+    return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size);
+}
+
+static int v4l2_request_vp9_end_frame(AVCodecContext *avctx)
+{
+    const VP9SharedContext *h = avctx->priv_data;
+    V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_VP9_FRAME,
+            .ptr = &controls->frame,
+            .size = sizeof(controls->frame),
+        },
+        {
+            .id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR,
+            .ptr = &controls->chp,
+            .size = sizeof(controls->chp),
+        },
+    };
+
+    return ff_v4l2_request_decode_frame(avctx, &controls->pic,
+                                        control, FF_ARRAY_ELEMS(control));
+}
+
+static int v4l2_request_vp9_init(AVCodecContext *avctx)
+{
+    struct v4l2_ctrl_vp9_frame frame;
+
+    struct v4l2_ext_control control[] = {
+        {
+            .id = V4L2_CID_STATELESS_VP9_FRAME,
+            .ptr = &frame,
+            .size = sizeof(frame),
+        },
+    };
+
+    fill_frame(&frame, avctx);
+
+    // TODO: check V4L2_CID_MPEG_VIDEO_VP9_PROFILE
+    // TODO: check V4L2_CID_MPEG_VIDEO_VP9_LEVEL
+
+    return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP9_FRAME,
+                                3 * 1024 * 1024,
+                                control, FF_ARRAY_ELEMS(control));
+}
+
+const FFHWAccel ff_vp9_v4l2request_hwaccel = {
+    .p.name             = "vp9_v4l2request",
+    .p.type             = AVMEDIA_TYPE_VIDEO,
+    .p.id               = AV_CODEC_ID_VP9,
+    .p.pix_fmt          = AV_PIX_FMT_DRM_PRIME,
+    .start_frame        = v4l2_request_vp9_start_frame,
+    .decode_slice       = v4l2_request_vp9_decode_slice,
+    .end_frame          = v4l2_request_vp9_end_frame,
+    .flush              = ff_v4l2_request_flush,
+    .frame_priv_data_size = sizeof(V4L2RequestControlsVP9),
+    .init               = v4l2_request_vp9_init,
+    .uninit             = ff_v4l2_request_uninit,
+    .priv_data_size     = sizeof(V4L2RequestContext),
+    .frame_params       = ff_v4l2_request_frame_params,
+};
diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 8ede2e2eb3..17b611bfdd 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -163,6 +163,7 @@ static int update_size(AVCodecContext *avctx, int w, int h)
                      CONFIG_VP9_D3D11VA_HWACCEL * 2 + \
                      CONFIG_VP9_D3D12VA_HWACCEL + \
                      CONFIG_VP9_NVDEC_HWACCEL + \
+                     CONFIG_VP9_V4L2REQUEST_HWACCEL + \
                      CONFIG_VP9_VAAPI_HWACCEL + \
                      CONFIG_VP9_VDPAU_HWACCEL + \
                      CONFIG_VP9_VIDEOTOOLBOX_HWACCEL)
@@ -202,6 +203,9 @@ static int update_size(AVCodecContext *avctx, int w, int h)
 #endif
 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
             *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX;
+#endif
+#if CONFIG_VP9_V4L2REQUEST_HWACCEL
+            *fmtp++ = AV_PIX_FMT_DRM_PRIME;
 #endif
             break;
         case AV_PIX_FMT_YUV420P12:
@@ -375,7 +379,7 @@ static av_always_inline int inv_recenter_nonneg(int v, int m)
 }
 
 // differential forward probability updates
-static int update_prob(VPXRangeCoder *c, int p)
+static int read_prob_delta(VPXRangeCoder *c)
 {
     static const uint8_t inv_map_table[255] = {
           7,  20,  33,  46,  59,  72,  85,  98, 111, 124, 137, 150, 163, 176,
@@ -429,8 +433,13 @@ static int update_prob(VPXRangeCoder *c, int p)
         av_assert2(d < FF_ARRAY_ELEMS(inv_map_table));
     }
 
-    return p <= 128 ? 1 + inv_recenter_nonneg(inv_map_table[d], p - 1) :
-                    255 - inv_recenter_nonneg(inv_map_table[d], 255 - p);
+    return inv_map_table[d];
+}
+
+static int update_prob(int p, int d)
+{
+    return p <= 128 ? 1 + inv_recenter_nonneg(d, p - 1) :
+                    255 - inv_recenter_nonneg(d, 255 - p);
 }
 
 static int read_colorspace_details(AVCodecContext *avctx)
@@ -697,7 +706,8 @@ static int decode_frame_header(AVCodecContext *avctx,
                                          get_bits(&s->gb, 8) : 255;
         }
 
-        if (get_bits1(&s->gb)) {
+        s->s.h.segmentation.update_data = get_bits1(&s->gb);
+        if (s->s.h.segmentation.update_data) {
             s->s.h.segmentation.absolute_vals = get_bits1(&s->gb);
             for (i = 0; i < 8; i++) {
                 if ((s->s.h.segmentation.feat[i].q_enabled = get_bits1(&s->gb)))
@@ -903,6 +913,8 @@ static int decode_frame_header(AVCodecContext *avctx,
      * as explicit copies if the fw update is missing (and skip the copy upon
      * fw update)? */
     s->prob.p = s->prob_ctx[c].p;
+    memset(&s->prob_raw.p, 0, sizeof(s->prob_raw.p));
+    memset(&s->prob_raw.coef, 0, sizeof(s->prob_raw.coef));
 
     // txfm updates
     if (s->s.h.lossless) {
@@ -914,18 +926,25 @@ static int decode_frame_header(AVCodecContext *avctx,
 
         if (s->s.h.txfmmode == TX_SWITCHABLE) {
             for (i = 0; i < 2; i++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.tx8p[i] = update_prob(&s->c, s->prob.p.tx8p[i]);
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.tx8p[i] = read_prob_delta(&s->c);
+                    s->prob.p.tx8p[i] = update_prob(s->prob.p.tx8p[i],
+                                                    s->prob_raw.p.tx8p[i]);
+                }
             for (i = 0; i < 2; i++)
                 for (j = 0; j < 2; j++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
-                        s->prob.p.tx16p[i][j] =
-                            update_prob(&s->c, s->prob.p.tx16p[i][j]);
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.tx16p[i][j] = read_prob_delta(&s->c);
+                        s->prob.p.tx16p[i][j] = update_prob(s->prob.p.tx16p[i][j],
+                                                            s->prob_raw.p.tx16p[i][j]);
+                    }
             for (i = 0; i < 2; i++)
                 for (j = 0; j < 3; j++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
-                        s->prob.p.tx32p[i][j] =
-                            update_prob(&s->c, s->prob.p.tx32p[i][j]);
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.tx32p[i][j] = read_prob_delta(&s->c);
+                        s->prob.p.tx32p[i][j] = update_prob(s->prob.p.tx32p[i][j],
+                                                            s->prob_raw.p.tx32p[i][j]);
+                    }
         }
     }
 
@@ -937,15 +956,18 @@ static int decode_frame_header(AVCodecContext *avctx,
                 for (k = 0; k < 2; k++)
                     for (l = 0; l < 6; l++)
                         for (m = 0; m < 6; m++) {
+                            uint8_t *pd = s->prob_raw.coef[i][j][k][l][m];
                             uint8_t *p = s->prob.coef[i][j][k][l][m];
                             uint8_t *r = ref[j][k][l][m];
                             if (m >= 3 && l == 0) // dc only has 3 pt
                                 break;
                             for (n = 0; n < 3; n++) {
-                                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                                    p[n] = update_prob(&s->c, r[n]);
-                                else
+                                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                                    pd[n] = read_prob_delta(&s->c);
+                                    p[n] = update_prob(r[n], pd[n]);
+                                } else {
                                     p[n] = r[n];
+                                }
                             }
                             memcpy(&p[3], ff_vp9_model_pareto8[p[2]], 8);
                         }
@@ -968,25 +990,37 @@ static int decode_frame_header(AVCodecContext *avctx,
 
     // mode updates
     for (i = 0; i < 3; i++)
-        if (vpx_rac_get_prob_branchy(&s->c, 252))
-            s->prob.p.skip[i] = update_prob(&s->c, s->prob.p.skip[i]);
+        if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+            s->prob_raw.p.skip[i] = read_prob_delta(&s->c);
+            s->prob.p.skip[i] = update_prob(s->prob.p.skip[i],
+                                            s->prob_raw.p.skip[i]);
+        }
     if (!s->s.h.keyframe && !s->s.h.intraonly) {
         for (i = 0; i < 7; i++)
             for (j = 0; j < 3; j++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_mode[i][j] = read_prob_delta(&s->c);
                     s->prob.p.mv_mode[i][j] =
-                        update_prob(&s->c, s->prob.p.mv_mode[i][j]);
+                        update_prob(s->prob.p.mv_mode[i][j],
+                                    s->prob_raw.p.mv_mode[i][j]);
+                }
 
         if (s->s.h.filtermode == FILTER_SWITCHABLE)
             for (i = 0; i < 4; i++)
                 for (j = 0; j < 2; j++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.filter[i][j] = read_prob_delta(&s->c);
                         s->prob.p.filter[i][j] =
-                            update_prob(&s->c, s->prob.p.filter[i][j]);
+                            update_prob(s->prob.p.filter[i][j],
+                                        s->prob_raw.p.filter[i][j]);
+                    }
 
         for (i = 0; i < 4; i++)
-            if (vpx_rac_get_prob_branchy(&s->c, 252))
-                s->prob.p.intra[i] = update_prob(&s->c, s->prob.p.intra[i]);
+            if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                s->prob_raw.p.intra[i] = read_prob_delta(&s->c);
+                s->prob.p.intra[i] = update_prob(s->prob.p.intra[i],
+                                                 s->prob_raw.p.intra[i]);
+            }
 
         if (s->s.h.allowcompinter) {
             s->s.h.comppredmode = vp89_rac_get(&s->c);
@@ -994,92 +1028,134 @@ static int decode_frame_header(AVCodecContext *avctx,
                 s->s.h.comppredmode += vp89_rac_get(&s->c);
             if (s->s.h.comppredmode == PRED_SWITCHABLE)
                 for (i = 0; i < 5; i++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.comp[i] = read_prob_delta(&s->c);
                         s->prob.p.comp[i] =
-                            update_prob(&s->c, s->prob.p.comp[i]);
+                            update_prob(s->prob.p.comp[i], s->prob_raw.p.comp[i]);
+                    }
         } else {
             s->s.h.comppredmode = PRED_SINGLEREF;
         }
 
         if (s->s.h.comppredmode != PRED_COMPREF) {
             for (i = 0; i < 5; i++) {
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.single_ref[i][0] = read_prob_delta(&s->c);
                     s->prob.p.single_ref[i][0] =
-                        update_prob(&s->c, s->prob.p.single_ref[i][0]);
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
+                        update_prob(s->prob.p.single_ref[i][0],
+                                    s->prob_raw.p.single_ref[i][0]);
+                }
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.single_ref[i][1] = read_prob_delta(&s->c);
                     s->prob.p.single_ref[i][1] =
-                        update_prob(&s->c, s->prob.p.single_ref[i][1]);
+                        update_prob(s->prob.p.single_ref[i][1],
+                                    s->prob_raw.p.single_ref[i][1]);
+                }
             }
         }
 
         if (s->s.h.comppredmode != PRED_SINGLEREF) {
             for (i = 0; i < 5; i++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.comp_ref[i] = read_prob_delta(&s->c);
                     s->prob.p.comp_ref[i] =
-                        update_prob(&s->c, s->prob.p.comp_ref[i]);
+                        update_prob(s->prob.p.comp_ref[i],
+                                    s->prob_raw.p.comp_ref[i]);
+                }
         }
 
         for (i = 0; i < 4; i++)
             for (j = 0; j < 9; j++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.y_mode[i][j] = read_prob_delta(&s->c);
                     s->prob.p.y_mode[i][j] =
-                        update_prob(&s->c, s->prob.p.y_mode[i][j]);
+                        update_prob(s->prob.p.y_mode[i][j],
+                                    s->prob_raw.p.y_mode[i][j]);
+                }
 
         for (i = 0; i < 4; i++)
             for (j = 0; j < 4; j++)
                 for (k = 0; k < 3; k++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.partition[i][j][k] = read_prob_delta(&s->c);
                         s->prob.p.partition[3 - i][j][k] =
-                            update_prob(&s->c,
-                                        s->prob.p.partition[3 - i][j][k]);
+                            update_prob(s->prob.p.partition[3 - i][j][k],
+                                        s->prob_raw.p.partition[i][j][k]);
+                    }
 
         // mv fields don't use the update_prob subexp model for some reason
         for (i = 0; i < 3; i++)
-            if (vpx_rac_get_prob_branchy(&s->c, 252))
-                s->prob.p.mv_joint[i] = (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+            if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                s->prob_raw.p.mv_joint[i] = (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                s->prob.p.mv_joint[i] = s->prob_raw.p.mv_joint[i];
+            }
 
         for (i = 0; i < 2; i++) {
-            if (vpx_rac_get_prob_branchy(&s->c, 252))
-                s->prob.p.mv_comp[i].sign =
+            if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                s->prob_raw.p.mv_comp[i].sign =
                     (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                s->prob.p.mv_comp[i].sign =
+                    s->prob_raw.p.mv_comp[i].sign;
+            }
 
             for (j = 0; j < 10; j++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.mv_comp[i].classes[j] =
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_comp[i].classes[j] =
                         (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                    s->prob.p.mv_comp[i].classes[j] =
+                        s->prob_raw.p.mv_comp[i].classes[j];
+                }
 
-            if (vpx_rac_get_prob_branchy(&s->c, 252))
-                s->prob.p.mv_comp[i].class0 =
+            if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                s->prob_raw.p.mv_comp[i].class0 =
                     (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                s->prob.p.mv_comp[i].class0 =
+                    s->prob_raw.p.mv_comp[i].class0;
+            }
 
             for (j = 0; j < 10; j++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.mv_comp[i].bits[j] =
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_comp[i].bits[j] =
                         (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                    s->prob.p.mv_comp[i].bits[j] =
+                        s->prob_raw.p.mv_comp[i].bits[j];
+                }
         }
 
         for (i = 0; i < 2; i++) {
             for (j = 0; j < 2; j++)
                 for (k = 0; k < 3; k++)
-                    if (vpx_rac_get_prob_branchy(&s->c, 252))
-                        s->prob.p.mv_comp[i].class0_fp[j][k] =
+                    if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                        s->prob_raw.p.mv_comp[i].class0_fp[j][k] =
                             (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                        s->prob.p.mv_comp[i].class0_fp[j][k] =
+                            s->prob_raw.p.mv_comp[i].class0_fp[j][k];
+                    }
 
             for (j = 0; j < 3; j++)
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.mv_comp[i].fp[j] =
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_comp[i].fp[j] =
                         (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                    s->prob.p.mv_comp[i].fp[j] =
+                        s->prob_raw.p.mv_comp[i].fp[j];
+                }
         }
 
         if (s->s.h.highprecisionmvs) {
             for (i = 0; i < 2; i++) {
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.mv_comp[i].class0_hp =
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_comp[i].class0_hp =
                         (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                    s->prob.p.mv_comp[i].class0_hp =
+                        s->prob_raw.p.mv_comp[i].class0_hp;
+                }
 
-                if (vpx_rac_get_prob_branchy(&s->c, 252))
-                    s->prob.p.mv_comp[i].hp =
+                if (vpx_rac_get_prob_branchy(&s->c, 252)) {
+                    s->prob_raw.p.mv_comp[i].hp =
                         (vp89_rac_get_uint(&s->c, 7) << 1) | 1;
+                    s->prob.p.mv_comp[i].hp =
+                        s->prob_raw.p.mv_comp[i].hp;
+                }
             }
         }
     }
@@ -1870,6 +1946,9 @@ const FFCodec ff_vp9_decoder = {
 #endif
 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
                                HWACCEL_VIDEOTOOLBOX(vp9),
+#endif
+#if CONFIG_VP9_V4L2REQUEST_HWACCEL
+                               HWACCEL_V4L2REQUEST(vp9),
 #endif
                                NULL
                            },
diff --git a/libavcodec/vp9dec.h b/libavcodec/vp9dec.h
index 81dc801052..99df1cb730 100644
--- a/libavcodec/vp9dec.h
+++ b/libavcodec/vp9dec.h
@@ -134,6 +134,10 @@ typedef struct VP9Context {
         ProbContext p;
         uint8_t coef[4][2][2][6][6][11];
     } prob;
+    struct {
+        ProbContext p;
+        uint8_t coef[4][2][2][6][6][11];
+    } prob_raw;
 
     // contextual (above) cache
     uint8_t *above_partition_ctx;
diff --git a/libavcodec/vp9shared.h b/libavcodec/vp9shared.h
index 8a450c26a6..37e389e9e1 100644
--- a/libavcodec/vp9shared.h
+++ b/libavcodec/vp9shared.h
@@ -136,6 +136,7 @@ typedef struct VP9BitstreamHeader {
         uint8_t temporal;
         uint8_t absolute_vals;
         uint8_t update_map;
+        uint8_t update_data;
         uint8_t prob[7];
         uint8_t pred_prob[3];
         struct {

From 904a85173fab816bb3c30652300efa93f2333657 Mon Sep 17 00:00:00 2001
From: Alex Bee <knaerzche@gmail.com>
Date: Sun, 15 Dec 2024 01:16:13 +0100
Subject: [PATCH 11/11] HACK: avcodec: Don't check VIDIOC_ENUM_FRAMESIZES
 step_width / step_height

Currently required, since hantro and rkvdec drivers expose wrong values.

Signed-off-by: Alex Bee <knaerzche@gmail.com>
---
 libavcodec/v4l2_request_probe.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libavcodec/v4l2_request_probe.c b/libavcodec/v4l2_request_probe.c
index 941042ec04..b3b1af3744 100644
--- a/libavcodec/v4l2_request_probe.c
+++ b/libavcodec/v4l2_request_probe.c
@@ -258,9 +258,7 @@ static int v4l2_request_try_framesize(AVCodecContext *avctx,
         } else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE ||
                     frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) &&
                    avctx->coded_width <= frmsize.stepwise.max_width &&
-                   avctx->coded_height <= frmsize.stepwise.max_height &&
-                   avctx->coded_width % frmsize.stepwise.step_width == 0 &&
-                   avctx->coded_height % frmsize.stepwise.step_height == 0) {
+                   avctx->coded_height <= frmsize.stepwise.max_height) {
             return 0;
         }
 
