Traneptora/FFmpeg
1
Fork 0
forked from FFmpeg/FFmpeg
Code Pull requests Activity

fftools/ffmpeg: add loopback decoding

Browse source 
This allows to send an encoder's output back to decoding and feed the
result into a complex filtergraph.
This commit is contained in:
Anton Khirnov 2024-02-19 10:27:44 +01:00
parent b98af440c5
commit a9193f7b7d
10 changed files with 336 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Changelog
View file

@ -33,6 +33,7 @@ version <next>:
- ffprobe -show_stream_groups option
- ffprobe (with -export_side_data film_grain) now prints film grain metadata
- AEA muxer
- ffmpeg CLI loopback decoders
version 6.1:

49
doc/ffmpeg.texi
View file

@ -219,6 +219,40 @@ Since there is no decoding or encoding, it is very fast and there is no quality
loss. However, it might not work in some cases because of many factors. Applying
filters is obviously also impossible, since filters work on uncompressed data.
@section Loopback decoders
While decoders are normally associated with demuxer streams, it is also possible
to create "loopback" decoders that decode the output from some encoder and allow
it to be fed back to complex filtergraphs. This is done with the @code{-dec}
directive, which takes as a parameter the index of the output stream that should
be decoded. Every such directive creates a new loopback decoder, indexed with
successive integers starting at zero. These indices should then be used to refer
to loopback decoders in complex filtergraph link labels, as described in the
documentation for @option{-filter_complex}.
E.g. the following example:
@example
ffmpeg -i INPUT \
-map 0:v:0 -c:v libx264 -crf 45 -f null - \
-dec 0:0 -filter_complex '[0:v][dec:0]hstack[stack]' \
-map '[stack]' -c:v ffv1 OUTPUT
@end example
reads an input video and
@itemize
@item
(line 2) encodes it with @code{libx264} at low quality;
@item
(line 3) decodes this encoded stream and places it side by side with the
original input video;
@item
(line 4) combined video is then losslessly encoded and written into
@file{OUTPUT}.
@end itemize
@c man end DETAILED DESCRIPTION
@chapter Stream selection
@ -2105,11 +2139,16 @@ type -- see the @option{-filter} options. @var{filtergraph} is a description of
the filtergraph, as described in the ``Filtergraph syntax'' section of the
ffmpeg-filters manual.
Input link labels must refer to input streams using the
@code{[file_index:stream_specifier]} syntax (i.e. the same as @option{-map}
uses). If @var{stream_specifier} matches multiple streams, the first one will be
used. An unlabeled input will be connected to the first unused input stream of
the matching type.
Input link labels must refer to either input streams or loopback decoders. For
input streams, use the @code{[file_index:stream_specifier]} syntax (i.e. the
same as @option{-map} uses). If @var{stream_specifier} matches multiple streams,
the first one will be used.
For decoders, the link label must be [dec:@var{dec_idx}], where @var{dec_idx} is
the index of the loopback decoder to be connected to given input.
An unlabeled input will be connected to the first unused input stream of the
matching type.
Output link labels are referred to with @option{-map}. Unlabeled outputs are
added to the first output file.

2
fftools/cmdutils.c
View file

@ -528,7 +528,7 @@ static void check_options(const OptionDef *po)
{
while (po->name) {
if (po->flags & OPT_PERFILE)
av_assert0(po->flags & (OPT_INPUT | OPT_OUTPUT));
av_assert0(po->flags & (OPT_INPUT | OPT_OUTPUT | OPT_DECODER));
if (po->type == OPT_TYPE_FUNC)
av_assert0(!(po->flags & (OPT_FLAG_OFFSET | OPT_FLAG_SPEC)));

7
fftools/cmdutils.h
View file

@ -144,8 +144,8 @@ typedef struct OptionDef {
#define OPT_AUDIO (1 << 4)
#define OPT_SUBTITLE (1 << 5)
#define OPT_DATA (1 << 6)
/* The option is per-file (currently ffmpeg-only). At least one of OPT_INPUT or
* OPT_OUTPUT must be set when this flag is in use.
/* The option is per-file (currently ffmpeg-only). At least one of OPT_INPUT,
* OPT_OUTPUT, OPT_DECODER must be set when this flag is in use.
*/
#define OPT_PERFILE (1 << 7)
@ -175,6 +175,9 @@ typedef struct OptionDef {
* name is stored in u1.name_canon */
#define OPT_HAS_CANON (1 << 14)
/* ffmpeg-only - OPT_PERFILE may apply to standalone decoders */
#define OPT_DECODER (1 << 15)
union {
void *dst_ptr;
int (*func_arg)(void *, const char *, const char *);

26
fftools/ffmpeg.c
View file

@ -131,6 +131,9 @@ int nb_output_files = 0;
FilterGraph **filtergraphs;
int nb_filtergraphs;
Decoder **decoders;
int nb_decoders;
#if HAVE_TERMIOS_H
/* init terminal so that we can grab keys */
@ -340,6 +343,10 @@ static void ffmpeg_cleanup(int ret)
for (int i = 0; i < nb_input_files; i++)
ifile_close(&input_files[i]);
for (int i = 0; i < nb_decoders; i++)
dec_free(&decoders[i]);
av_freep(&decoders);
if (vstats_file) {
if (fclose(vstats_file))
av_log(NULL, AV_LOG_ERROR,
@ -404,6 +411,10 @@ InputStream *ist_iter(InputStream *prev)
static void frame_data_free(void *opaque, uint8_t *data)
{
FrameData *fd = (FrameData *)data;
avcodec_parameters_free(&fd->par_enc);
av_free(data);
}
@ -430,6 +441,21 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
const FrameData *fd_src = (const FrameData *)src->data;
memcpy(fd, fd_src, sizeof(*fd));
fd->par_enc = NULL;
if (fd_src->par_enc) {
int ret = 0;
fd->par_enc = avcodec_parameters_alloc();
ret = fd->par_enc ?
avcodec_parameters_copy(fd->par_enc, fd_src->par_enc) :
AVERROR(ENOMEM);
if (ret < 0) {
av_buffer_unref(dst);
av_buffer_unref(&src);
return ret;
}
}
av_buffer_unref(&src);
} else {

22
fftools/ffmpeg.h
View file

@ -331,6 +331,8 @@ typedef struct DecoderOpts {
typedef struct Decoder {
const AVClass *class;
enum AVMediaType type;
const uint8_t *subtitle_header;
int subtitle_header_size;
@ -606,6 +608,8 @@ typedef struct FrameData {
int bits_per_raw_sample;
int64_t wallclock[LATENCY_PROBE_NB];
AVCodecParameters *par_enc;
} FrameData;
extern InputFile **input_files;
@ -617,6 +621,10 @@ extern int nb_output_files;
extern FilterGraph **filtergraphs;
extern int nb_filtergraphs;
// standalone decoders (not tied to demuxed streams)
extern Decoder **decoders;
extern int nb_decoders;
extern char *vstats_filename;
extern float dts_delta_threshold;
@ -734,6 +742,11 @@ void hw_device_free_all(void);
*/
AVBufferRef *hw_device_for_filter(void);
/**
* Create a standalone decoder.
*/
int dec_create(const OptionsContext *o, const char *arg, Scheduler *sch);
/**
* @param dec_opts Dictionary filled with decoder options. Its ownership
* is transferred to the decoder.
@ -748,12 +761,21 @@ int dec_init(Decoder **pdec, Scheduler *sch,
AVFrame *param_out);
void dec_free(Decoder **pdec);
/*
* Called by filters to connect decoder's output to given filtergraph input.
*
* @param opts filtergraph input options, to be filled by this function
*/
int dec_filter_add(Decoder *dec, InputFilter *ifilter, InputFilterOptions *opts);
int enc_alloc(Encoder **penc, const AVCodec *codec,
Scheduler *sch, unsigned sch_idx);
void enc_free(Encoder **penc);
int enc_open(void *opaque, const AVFrame *frame);
int enc_loopback(Encoder *enc);
/*
* Initialize muxing state for the given stream, should be called
* after the codec/streamcopy setup has been done.

164
fftools/ffmpeg_dec.c
View file

@ -17,6 +17,7 @@
*/
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/dict.h"
#include "libavutil/error.h"
#include "libavutil/log.h"
@ -71,9 +72,16 @@ typedef struct DecoderPriv {
Scheduler *sch;
unsigned sch_idx;
// this decoder's index in decoders or -1
int index;
void *log_parent;
char log_name[32];
char *parent_name;
struct {
AVDictionary *opts;
const AVCodec *codec;
} standalone_init;
} DecoderPriv;
static DecoderPriv *dp_from_dec(Decoder *d)
@ -101,6 +109,8 @@ void dec_free(Decoder **pdec)
av_frame_free(&dp->frame);
av_packet_free(&dp->pkt);
av_dict_free(&dp->standalone_init.opts);
for (int i = 0; i < FF_ARRAY_ELEMS(dp->sub_prev); i++)
av_frame_free(&dp->sub_prev[i]);
av_frame_free(&dp->sub_heartbeat);
@ -145,6 +155,7 @@ static int dec_alloc(DecoderPriv **pdec, Scheduler *sch, int send_end_ts)
if (!dp->pkt)
goto fail;
dp->index = -1;
dp->dec.class = &dec_class;
dp->last_filter_in_rescale_delta = AV_NOPTS_VALUE;
dp->last_frame_pts = AV_NOPTS_VALUE;
@ -754,11 +765,56 @@ static int packet_decode(DecoderPriv *dp, AVPacket *pkt, AVFrame *frame)
}
}
static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts,
const DecoderOpts *o, AVFrame *param_out);
static int dec_standalone_open(DecoderPriv *dp, const AVPacket *pkt)
{
DecoderOpts o;
const FrameData *fd;
char name[16];
if (!pkt->opaque_ref)
return AVERROR_BUG;
fd = (FrameData *)pkt->opaque_ref->data;
if (!fd->par_enc)
return AVERROR_BUG;
memset(&o, 0, sizeof(o));
o.par = fd->par_enc;
o.time_base = pkt->time_base;
o.codec = dp->standalone_init.codec;
if (!o.codec)
o.codec = avcodec_find_decoder(o.par->codec_id);
if (!o.codec) {
const AVCodecDescriptor *desc = avcodec_descriptor_get(o.par->codec_id);
av_log(dp, AV_LOG_ERROR, "Cannot find a decoder for codec ID '%s'\n",
desc ? desc->name : "?");
return AVERROR_DECODER_NOT_FOUND;
}
snprintf(name, sizeof(name), "dec%d", dp->index);
o.name = name;
return dec_open(dp, &dp->standalone_init.opts, &o, NULL);
}
static void dec_thread_set_name(const DecoderPriv *dp)
{
char name[16];
snprintf(name, sizeof(name), "dec%s:%s", dp->parent_name,
dp->dec_ctx->codec->name);
char name[16] = "dec";
if (dp->index >= 0)
av_strlcatf(name, sizeof(name), "%d", dp->index);
else if (dp->parent_name)
av_strlcat(name, dp->parent_name, sizeof(name));
if (dp->dec_ctx)
av_strlcatf(name, sizeof(name), ":%s", dp->dec_ctx->codec->name);
ff_thread_setname(name);
}
@ -814,6 +870,22 @@ static int decoder_thread(void *arg)
av_log(dp, AV_LOG_VERBOSE, "Decoder thread received %s packet\n",
flush_buffers ? "flush" : "EOF");
// this is a standalone decoder that has not been initialized yet
if (!dp->dec_ctx) {
if (flush_buffers)
continue;
if (input_status < 0) {
av_log(dp, AV_LOG_ERROR,
"Cannot initialize a standalone decoder\n");
ret = input_status;
goto finish;
}
ret = dec_standalone_open(dp, dt.pkt);
if (ret < 0)
goto finish;
}
ret = packet_decode(dp, have_data ? dt.pkt : NULL, dt.frame);
av_packet_unref(dt.pkt);
@ -1075,6 +1147,7 @@ static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts,
dp->flags = o->flags;
dp->log_parent = o->log_parent;
dp->dec.type = codec->type;
dp->framerate_in = o->framerate;
dp->hwaccel_id = o->hwaccel_id;
@ -1188,3 +1261,88 @@ fail:
dec_free((Decoder**)&dp);
return ret;
}
int dec_create(const OptionsContext *o, const char *arg, Scheduler *sch)
{
DecoderPriv *dp;
OutputFile *of;
OutputStream *ost;
int of_index, ost_index;
char *p;
unsigned enc_idx;
int ret;
ret = dec_alloc(&dp, sch, 0);
if (ret < 0)
return ret;
dp->index = nb_decoders;
ret = GROW_ARRAY(decoders, nb_decoders);
if (ret < 0) {
dec_free((Decoder **)&dp);
return ret;
}
decoders[nb_decoders - 1] = (Decoder *)dp;
of_index = strtol(arg, &p, 0);
if (of_index < 0 || of_index >= nb_output_files) {
av_log(dp, AV_LOG_ERROR, "Invalid output file index '%d' in %s\n", of_index, arg);
return AVERROR(EINVAL);
}
of = output_files[of_index];
ost_index = strtol(p + 1, NULL, 0);
if (ost_index < 0 || ost_index >= of->nb_streams) {
av_log(dp, AV_LOG_ERROR, "Invalid output stream index '%d' in %s\n", ost_index, arg);
return AVERROR(EINVAL);
}
ost = of->streams[ost_index];
if (!ost->enc) {
av_log(dp, AV_LOG_ERROR, "Output stream %s has no encoder\n", arg);
return AVERROR(EINVAL);
}
dp->dec.type = ost->type;
ret = enc_loopback(ost->enc);
if (ret < 0)
return ret;
enc_idx = ret;
ret = sch_connect(sch, SCH_ENC(enc_idx), SCH_DEC(dp->sch_idx));
if (ret < 0)
return ret;
ret = av_dict_copy(&dp->standalone_init.opts, o->g->codec_opts, 0);
if (ret < 0)
return ret;
if (o->codec_names.nb_opt) {
const char *name = o->codec_names.opt[o->codec_names.nb_opt - 1].u.str;
dp->standalone_init.codec = avcodec_find_decoder_by_name(name);
if (!dp->standalone_init.codec) {
av_log(dp, AV_LOG_ERROR, "No such decoder: %s\n", name);
return AVERROR_DECODER_NOT_FOUND;
}
}
return 0;
}
int dec_filter_add(Decoder *d, InputFilter *ifilter, InputFilterOptions *opts)
{
DecoderPriv *dp = dp_from_dec(d);
char name[16];
snprintf(name, sizeof(name), "dec%d", dp->index);
opts->name = av_strdup(name);
if (!opts->name)
return AVERROR(ENOMEM);
return dp->sch_idx;
}

19
fftools/ffmpeg_enc.c
View file

@ -49,6 +49,7 @@ struct Encoder {
uint64_t packets_encoded;
int opened;
int attach_par;
Scheduler *sch;
unsigned sch_idx;
@ -693,6 +694,18 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
return AVERROR(ENOMEM);
fd->wallclock[LATENCY_PROBE_ENC_POST] = av_gettime_relative();
// attach stream parameters to first packet if requested
avcodec_parameters_free(&fd->par_enc);
if (e->attach_par && !e->packets_encoded) {
fd->par_enc = avcodec_parameters_alloc();
if (!fd->par_enc)
return AVERROR(ENOMEM);
ret = avcodec_parameters_from_context(fd->par_enc, enc);
if (ret < 0)
return ret;
}
if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
ret = update_video_stats(ost, pkt, !!vstats_filename);
if (ret < 0)
@ -929,3 +942,9 @@ finish:
return ret;
}
int enc_loopback(Encoder *enc)
{
enc->attach_par = 1;
return enc->sch_idx;
}

47
fftools/ffmpeg_filter.c
View file

@ -710,6 +710,34 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
return 0;
}
static int ifilter_bind_dec(InputFilterPriv *ifp, Decoder *dec)
{
FilterGraphPriv *fgp = fgp_from_fg(ifp->ifilter.graph);
int ret, dec_idx;
av_assert0(!ifp->bound);
ifp->bound = 1;
if (ifp->type != dec->type) {
av_log(fgp, AV_LOG_ERROR, "Tried to connect %s decoder to %s filtergraph input\n",
av_get_media_type_string(dec->type), av_get_media_type_string(ifp->type));
return AVERROR(EINVAL);
}
ifp->type_src = ifp->type;
dec_idx = dec_filter_add(dec, &ifp->ifilter, &ifp->opts);
if (dec_idx < 0)
return dec_idx;
ret = sch_connect(fgp->sch, SCH_DEC(dec_idx),
SCH_FILTER_IN(fgp->sch_idx, ifp->index));
if (ret < 0)
return ret;
return 0;
}
static int set_channel_layout(OutputFilterPriv *f, OutputStream *ost)
{
const AVCodec *c = ost->enc_ctx->codec;
@ -1114,7 +1142,24 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
enum AVMediaType type = ifp->type;
int i, ret;
if (ifp->linklabel) {
if (ifp->linklabel && !strncmp(ifp->linklabel, "dec:", 4)) {
// bind to a standalone decoder
int dec_idx;
dec_idx = strtol(ifp->linklabel + 4, NULL, 0);
if (dec_idx < 0 || dec_idx >= nb_decoders) {
av_log(fg, AV_LOG_ERROR, "Invalid decoder index %d in filtergraph description %s\n",
dec_idx, fgp->graph_desc);
return AVERROR(EINVAL);
}
ret = ifilter_bind_dec(ifp, decoders[dec_idx]);
if (ret < 0)
av_log(fg, AV_LOG_ERROR, "Error binding a decoder to filtergraph input %s\n",
ifilter->name);
return ret;
} else if (ifp->linklabel) {
// bind to an explicitly specified demuxer stream
AVFormatContext *s;
AVStream *st = NULL;
char *p;

13
fftools/ffmpeg_opt.c
View file

@ -1177,11 +1177,13 @@ void show_usage(void)
enum OptGroup {
GROUP_OUTFILE,
GROUP_INFILE,
GROUP_DECODER,
};
static const OptionGroupDef groups[] = {
[GROUP_OUTFILE] = { "output url", NULL, OPT_OUTPUT },
[GROUP_INFILE] = { "input url", "i", OPT_INPUT },
[GROUP_DECODER] = { "loopback decoder", "dec", OPT_DECODER },
};
static int open_files(OptionGroupList *l, const char *inout, Scheduler *sch,
@ -1259,6 +1261,13 @@ int ffmpeg_parse_options(int argc, char **argv, Scheduler *sch)
goto fail;
}
/* create loopback decoders */
ret = open_files(&octx.groups[GROUP_DECODER], "decoder", sch, dec_create);
if (ret < 0) {
errmsg = "creating loopback decoders";
goto fail;
}
// bind unbound filtegraph inputs/outputs and check consistency
for (int i = 0; i < nb_filtergraphs; i++) {
ret = fg_finalise_bindings(filtergraphs[i]);
@ -1367,11 +1376,11 @@ const OptionDef options[] = {
{ "recast_media", OPT_TYPE_BOOL, OPT_EXPERT,
{ &recast_media },
"allow recasting stream type in order to force a decoder of different media type" },
{ "c", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_HAS_CANON,
{ "c", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_DECODER | OPT_HAS_CANON,
{ .off = OFFSET(codec_names) },
"select encoder/decoder ('copy' to copy stream without reencoding)", "codec",
.u1.name_canon = "codec", },
{ "codec", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_EXPERT | OPT_HAS_ALT,
{ "codec", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_DECODER | OPT_EXPERT | OPT_HAS_ALT,
{ .off = OFFSET(codec_names) },
"alias for -c (select encoder/decoder)", "codec",
.u1.names_alt = alt_codec, },