From e61b9d4094d76f5fede0b65139d17add281e6e45 Mon Sep 17 00:00:00 2001 From: James Almer Date: Sun, 24 Mar 2024 17:26:20 -0300 Subject: [PATCH] fftools/ffmpeg: propagate decoded_side_data from decoded streams to the filterchain Global side data as exported by a decoder may no longer apply if a filter in the chain altered the frames in some form, like changing color, dimensions, or channel layout information. After this change, any such changes in side data will be taken into account by the encoder futher in the process. Signed-off-by: James Almer --- fftools/ffmpeg_dec.c | 10 +++++++ fftools/ffmpeg_filter.c | 60 +++++++++++++++++++++++++++++++++++++++++ fftools/ffmpeg_utils.h | 16 +++++++++++ 3 files changed, 86 insertions(+) diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c index 2723a0312e..84ba5c6d5d 100644 --- a/fftools/ffmpeg_dec.c +++ b/fftools/ffmpeg_dec.c @@ -788,6 +788,11 @@ static int packet_decode(DecoderPriv *dp, AVPacket *pkt, AVFrame *frame) frame->time_base = dec->pkt_timebase; + ret = clone_side_data(&frame->side_data, &frame->nb_side_data, + dec->decoded_side_data, dec->nb_decoded_side_data, 0); + if (ret < 0) + return ret; + if (dec->codec_type == AVMEDIA_TYPE_AUDIO) { dp->dec.samples_decoded += frame->nb_samples; @@ -1638,6 +1643,11 @@ static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts, param_out->color_range = dp->dec_ctx->color_range; } + av_frame_side_data_free(¶m_out->side_data, ¶m_out->nb_side_data); + ret = clone_side_data(¶m_out->side_data, ¶m_out->nb_side_data, + dp->dec_ctx->decoded_side_data, dp->dec_ctx->nb_decoded_side_data, 0); + if (ret < 0) + return ret; param_out->time_base = dp->dec_ctx->pkt_timebase; } diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c index 29797a0b71..0fb31243f8 100644 --- a/fftools/ffmpeg_filter.c +++ b/fftools/ffmpeg_filter.c @@ -139,6 +139,9 @@ typedef struct InputFilterPriv { AVRational time_base; + AVFrameSideData **side_data; + int nb_side_data; + AVFifo *frame_queue; AVBufferRef *hw_frames_ctx; @@ -205,6 +208,9 @@ typedef struct OutputFilterPriv { enum AVColorSpace color_space; enum AVColorRange color_range; + AVFrameSideData **side_data; + int nb_side_data; + // time base in which the output is sent to our downstream // does not need to match the filtersink's timebase AVRational tb_out; @@ -1009,6 +1015,7 @@ void fg_free(FilterGraph **pfg) av_buffer_unref(&ifp->hw_frames_ctx); av_freep(&ifp->linklabel); av_freep(&ifp->opts.name); + av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data); av_freep(&ifilter->name); av_freep(&fg->inputs[j]); } @@ -1026,6 +1033,7 @@ void fg_free(FilterGraph **pfg) av_freep(&ofilter->apad); av_freep(&ofp->name); av_channel_layout_uninit(&ofp->ch_layout); + av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data); av_freep(&fg->outputs[j]); } av_freep(&fg->outputs); @@ -1725,6 +1733,9 @@ static int configure_input_video_filter(FilterGraph *fg, AVFilterGraph *graph, par->color_space = ifp->color_space; par->color_range = ifp->color_range; par->hw_frames_ctx = ifp->hw_frames_ctx; + par->side_data = ifp->side_data; + par->nb_side_data = ifp->nb_side_data; + ret = av_buffersrc_parameters_set(ifp->filter, par); if (ret < 0) goto fail; @@ -1809,6 +1820,7 @@ static int configure_input_audio_filter(FilterGraph *fg, AVFilterGraph *graph, { InputFilterPriv *ifp = ifp_from_ifilter(ifilter); AVFilterContext *last_filter; + AVBufferSrcParameters *par; const AVFilter *abuffer_filt = avfilter_get_by_name("abuffer"); AVBPrint args; char name[255]; @@ -1831,6 +1843,15 @@ static int configure_input_audio_filter(FilterGraph *fg, AVFilterGraph *graph, name, args.str, NULL, graph)) < 0) return ret; + par = av_buffersrc_parameters_alloc(); + if (!par) + return AVERROR(ENOMEM); + par->side_data = ifp->side_data; + par->nb_side_data = ifp->nb_side_data; + ret = av_buffersrc_parameters_set(ifp->filter, par); + av_free(par); + if (ret < 0) + return ret; last_filter = ifp->filter; snprintf(name, sizeof(name), "trim for input stream %s", ifp->opts.name); @@ -1970,6 +1991,8 @@ static int configure_filtergraph(FilterGraph *fg, FilterGraphThread *fgt) /* limit the lists of allowed formats to the ones selected, to * make sure they stay the same if the filtergraph is reconfigured later */ for (int i = 0; i < fg->nb_outputs; i++) { + const AVFrameSideData *const *sd; + int nb_sd; OutputFilter *ofilter = fg->outputs[i]; OutputFilterPriv *ofp = ofp_from_ofilter(ofilter); AVFilterContext *sink = ofp->filter; @@ -1998,6 +2021,17 @@ static int configure_filtergraph(FilterGraph *fg, FilterGraphThread *fgt) ret = av_buffersink_get_ch_layout(sink, &ofp->ch_layout); if (ret < 0) goto fail; + av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data); + sd = av_buffersink_get_side_data(sink, &nb_sd); + if (nb_sd) + for (int j = 0; j < nb_sd; j++) { + int ret = av_frame_side_data_clone(&ofp->side_data, &ofp->nb_side_data, + sd[j], 0); + if (ret < 0) { + av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data); + goto fail; + } + } } for (int i = 0; i < fg->nb_inputs; i++) { @@ -2066,6 +2100,20 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr if (ret < 0) return ret; + av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data); + for (int i = 0; i < frame->nb_side_data; i++) { + const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type); + + if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL)) + continue; + + ret = av_frame_side_data_clone(&ifp->side_data, + &ifp->nb_side_data, + frame->side_data[i], 0); + if (ret < 0) + return ret; + } + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX); if (sd) memcpy(ifp->displaymatrix, sd->data, sizeof(ifp->displaymatrix)); @@ -2396,6 +2444,11 @@ static int close_output(OutputFilterPriv *ofp, FilterGraphThread *fgt) if (ret < 0) return ret; } + av_frame_side_data_free(&frame->side_data, &frame->nb_side_data); + ret = clone_side_data(&frame->side_data, &frame->nb_side_data, + ofp->side_data, ofp->nb_side_data, 0); + if (ret < 0) + return ret; fd = frame_data(frame); if (!fd) @@ -2748,6 +2801,13 @@ static int send_eof(FilterGraphThread *fgt, InputFilter *ifilter, if (ret < 0) return ret; + av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data); + ret = clone_side_data(&ifp->side_data, &ifp->nb_side_data, + ifp->opts.fallback->side_data, + ifp->opts.fallback->nb_side_data, 0); + if (ret < 0) + return ret; + if (ifilter_has_all_input_formats(ifilter->graph)) { ret = configure_filtergraph(ifilter->graph, fgt); if (ret < 0) { diff --git a/fftools/ffmpeg_utils.h b/fftools/ffmpeg_utils.h index 8ed6f81b28..9ca3afffa0 100644 --- a/fftools/ffmpeg_utils.h +++ b/fftools/ffmpeg_utils.h @@ -44,4 +44,20 @@ static inline int err_merge(int err0, int err1) return (err0 < 0) ? err0 : FFMIN(err1, 0); } +/** + * Wrapper calling av_frame_side_data_clone() in a loop for all source entries. + * It does not clear dst beforehand. */ +static inline int clone_side_data(AVFrameSideData ***dst, int *nb_dst, + AVFrameSideData * const *src, int nb_src, + unsigned int flags) +{ + for (int i = 0; i < nb_src; i++) { + int ret = av_frame_side_data_clone(dst, nb_dst, src[i], flags); + if (ret < 0) + return ret; + } + + return 0; +} + #endif // FFTOOLS_FFMPEG_UTILS_H