avcodec/h263dec: Remove AVCodec.pix_fmts arrays

They are not intended for decoders (for which there is the get_format
callback in case the user has a choice).

Also note that the list was wrong for MPEG4, because it did not contain
the high bit depth pixel formats used for studio profiles.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavcodec/flvdec.c

@@ -123,6 +123,4 @@ const FFCodec ff_flv_decoder = {
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P,
-                                                     AV_PIX_FMT_NONE },
 };

libavcodec/h263dec.c

@@ -49,6 +49,23 @@
 #include "thread.h"
 #include "wmv2dec.h"
 
+static const enum AVPixelFormat h263_hwaccel_pixfmt_list_420[] = {
+#if CONFIG_H263_VAAPI_HWACCEL || CONFIG_MPEG4_VAAPI_HWACCEL
+    AV_PIX_FMT_VAAPI,
+#endif
+#if CONFIG_MPEG4_NVDEC_HWACCEL
+    AV_PIX_FMT_CUDA,
+#endif
+#if CONFIG_MPEG4_VDPAU_HWACCEL
+    AV_PIX_FMT_VDPAU,
+#endif
+#if CONFIG_H263_VIDEOTOOLBOX_HWACCEL || CONFIG_MPEG4_VIDEOTOOLBOX_HWACCEL
+    AV_PIX_FMT_VIDEOTOOLBOX,
+#endif
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_NONE
+};
+
 static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
 {
     /* MPEG-4 Studio Profile only, not supported by hardware */
@@ -63,7 +80,12 @@ static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
         return AV_PIX_FMT_GRAY8;
     }
 
-    return avctx->pix_fmt = ff_get_format(avctx, avctx->codec->pix_fmts);
+    if (avctx->codec_id == AV_CODEC_ID_H263  ||
+        avctx->codec_id == AV_CODEC_ID_H263P ||
+        avctx->codec_id == AV_CODEC_ID_MPEG4)
+        return avctx->pix_fmt = ff_get_format(avctx, h263_hwaccel_pixfmt_list_420);
+
+    return AV_PIX_FMT_YUV420P;
 }
 
 av_cold int ff_h263_decode_init(AVCodecContext *avctx)
@@ -659,23 +681,6 @@ frame_end:
         return get_consumed_bytes(s, buf_size);
 }
 
-const enum AVPixelFormat ff_h263_hwaccel_pixfmt_list_420[] = {
-#if CONFIG_H263_VAAPI_HWACCEL || CONFIG_MPEG4_VAAPI_HWACCEL
-    AV_PIX_FMT_VAAPI,
-#endif
-#if CONFIG_MPEG4_NVDEC_HWACCEL
-    AV_PIX_FMT_CUDA,
-#endif
-#if CONFIG_MPEG4_VDPAU_HWACCEL
-    AV_PIX_FMT_VDPAU,
-#endif
-#if CONFIG_H263_VIDEOTOOLBOX_HWACCEL || CONFIG_MPEG4_VIDEOTOOLBOX_HWACCEL
-    AV_PIX_FMT_VIDEOTOOLBOX,
-#endif
-    AV_PIX_FMT_YUV420P,
-    AV_PIX_FMT_NONE
-};
-
 static const AVCodecHWConfigInternal *const h263_hw_config_list[] = {
 #if CONFIG_H263_VAAPI_HWACCEL
     HWACCEL_VAAPI(h263),
@@ -706,7 +711,6 @@ const FFCodec ff_h263_decoder = {
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .flush          = ff_mpeg_flush,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = ff_h263_hwaccel_pixfmt_list_420,
     .hw_configs     = h263_hw_config_list,
 };
 
@@ -724,6 +728,5 @@ const FFCodec ff_h263p_decoder = {
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .flush          = ff_mpeg_flush,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = ff_h263_hwaccel_pixfmt_list_420,
     .hw_configs     = h263_hw_config_list,
 };

libavcodec/h263dec.h

@@ -38,8 +38,6 @@ extern VLCElem ff_h263_inter_MCBPC_vlc[];
 extern VLCElem ff_h263_cbpy_vlc[];
 extern VLCElem ff_h263_mv_vlc[];
 
-extern const enum AVPixelFormat ff_h263_hwaccel_pixfmt_list_420[];
-
 int ff_h263_decode_motion(MpegEncContext * s, int pred, int f_code);
 int ff_h263_decode_init(AVCodecContext *avctx);
 int ff_h263_decode_frame(AVCodecContext *avctx, AVFrame *frame,

libavcodec/intelh263dec.c

@@ -138,8 +138,4 @@ const FFCodec ff_h263i_decoder = {
     FF_CODEC_DECODE_CB(ff_h263_decode_frame),
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) {
-        AV_PIX_FMT_YUV420P,
-        AV_PIX_FMT_NONE
-    },
 };

libavcodec/mpeg4videodec.c

@@ -3865,7 +3865,6 @@ const FFCodec ff_mpeg4_decoder = {
                              FF_CODEC_CAP_ALLOCATE_PROGRESS,
     .flush                 = ff_mpeg_flush,
     .p.max_lowres          = 3,
-    .p.pix_fmts            = ff_h263_hwaccel_pixfmt_list_420,
     .p.profiles            = NULL_IF_CONFIG_SMALL(ff_mpeg4_video_profiles),
     UPDATE_THREAD_CONTEXT(mpeg4_update_thread_context),
     UPDATE_THREAD_CONTEXT_FOR_USER(mpeg4_update_thread_context_for_user),

libavcodec/msmpeg4dec.c

@@ -853,10 +853,6 @@ const FFCodec ff_msmpeg4v1_decoder = {
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) {
-        AV_PIX_FMT_YUV420P,
-        AV_PIX_FMT_NONE
-    },
 };
 
 const FFCodec ff_msmpeg4v2_decoder = {
@@ -871,10 +867,6 @@ const FFCodec ff_msmpeg4v2_decoder = {
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) {
-        AV_PIX_FMT_YUV420P,
-        AV_PIX_FMT_NONE
-    },
 };
 
 const FFCodec ff_msmpeg4v3_decoder = {
@@ -889,10 +881,6 @@ const FFCodec ff_msmpeg4v3_decoder = {
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) {
-        AV_PIX_FMT_YUV420P,
-        AV_PIX_FMT_NONE
-    },
 };
 
 const FFCodec ff_wmv1_decoder = {
@@ -907,8 +895,4 @@ const FFCodec ff_wmv1_decoder = {
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_SKIP_FRAME_FILL_PARAM,
     .p.max_lowres   = 3,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) {
-        AV_PIX_FMT_YUV420P,
-        AV_PIX_FMT_NONE
-    },
 };

libavcodec/wmv2dec.c

@@ -599,6 +599,4 @@ const FFCodec ff_wmv2_decoder = {
     FF_CODEC_DECODE_CB(ff_h263_decode_frame),
     .p.capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1,
     .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
-    .p.pix_fmts     = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P,
-                                                     AV_PIX_FMT_NONE },
 };