From 4960acb627d736421251500b3e4e0e88d3b12cd1 Mon Sep 17 00:00:00 2001
From: John Cox <jc@kynesim.co.uk>
Date: Fri, 12 Jan 2024 16:46:27 +0000
Subject: [PATCH 02/14] v4l2_m2m_dec: Try to accomodate ffmpegs ideas about
 default s/w fmts

---
 libavcodec/v4l2_m2m_dec.c | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
index f67dd23ba1..3883223079 100644
--- a/libavcodec/v4l2_m2m_dec.c
+++ b/libavcodec/v4l2_m2m_dec.c
@@ -1079,7 +1079,6 @@ choose_capture_format(AVCodecContext * const avctx, V4L2m2mContext * const s)
     unsigned int fmts_n;
     uint32_t *fmts = ff_v4l2_context_enum_drm_formats(&s->capture, &fmts_n);
     enum AVPixelFormat *fmts2 = NULL;
-    enum AVPixelFormat t;
     enum AVPixelFormat gf_pix_fmt;
     unsigned int i;
     unsigned int n = 0;
@@ -1089,7 +1088,7 @@ choose_capture_format(AVCodecContext * const avctx, V4L2m2mContext * const s)
     if (!fmts)
         return AVERROR(ENOENT);
 
-    if ((fmts2 = av_malloc(sizeof(*fmts2) * (fmts_n + 2))) == NULL) {
+    if ((fmts2 = av_malloc(sizeof(*fmts2) * (fmts_n + 3))) == NULL) {
         rv = AVERROR(ENOMEM);
         goto error;
     }
@@ -1110,17 +1109,25 @@ choose_capture_format(AVCodecContext * const avctx, V4L2m2mContext * const s)
             pref_n = n;
         fmts2[n++] = f;
     }
-    fmts2[n] = AV_PIX_FMT_NONE;
 
     if (n < 2) {
         av_log(avctx, AV_LOG_DEBUG, "%s: No usable formats found\n", __func__);
         goto error;
     }
 
-    // Put preferred s/w format at the end - ff_get_format will put it in sw_pix_fmt
-    t = fmts2[n - 1];
-    fmts2[n - 1] = fmts2[pref_n];
-    fmts2[pref_n] = t;
+    if (n != 2) {
+        // ffmpeg.c really only expects one s/w format. It thinks that the
+        // last format in the list is the s/w format of the h/w format but
+        // also chooses the first non-h/w format as the preferred s/w format.
+        // The only way of reconciling this is to dup our preferred format into
+        // both last & first place :-(
+        const enum AVPixelFormat t = fmts2[pref_n];
+        fmts2[pref_n] = fmts2[1];
+        fmts2[1] = t;
+        fmts2[n++] = t;
+    }
+
+    fmts2[n] = AV_PIX_FMT_NONE;
 
     gf_pix_fmt = ff_get_format(avctx, fmts2);
     av_log(avctx, AV_LOG_DEBUG, "avctx requested=%d (%s) %dx%d; get_format requested=%d (%s)\n",
-- 
2.34.1

