ffmpeg 双路输入解析——以vf_overlay为例

ffmpeg里面处理两路输入的模块都放在libavfilter/dualinput.c里面
首先定义了一个context用于描述dualinput的信息

FFDualInputContext

typedef struct FFDualInputContext {    FFFrameSync fs;    AVFrame *(*process)(AVFilterContext *ctx, AVFrame *main, const AVFrame *second);    int shortest;               ///< terminate stream when the second input terminates    int repeatlast;             ///< repeat last second frame    int skip_initial_unpaired;  ///< Skip initial frames that do not have a 2nd input} FFDualInputContext;

process是一个函数指针，以vf_overlay为例，调用过程为

static AVFrame *do_blend(AVFilterContext *ctx, AVFrame *mainpic,                         const AVFrame *second){    ......}

static av_cold void uninit(AVFilterContext *ctx){    OverlayContext *s = ctx->priv;    ff_dualinput_uninit(&s->dinput);    ......}

 s->dinput.process = do_blend;

但是到这里还是不明白main和second怎么传入do_blend的
用gdb分析，先b do_blend再bt，得到的结果为

调用过程backtrace

(gdb) bt #0  do_blend (ctx=0x2b6d7a0, mainpic=0x2b6cb80, second=0x2fa7e00) at libavfilter/vf_overlay.c:772#1  0x000000000066e3dc in process_frame (fs=0x2e60708) at libavfilter/dualinput.c:37#2  0x000000000046fe07 in ff_framesync_process_frame (fs=0x2e60708, all=0) at libavfilter/framesync.c:291#3  0x000000000046fee5 in ff_framesync_filter_frame (fs=0x2e60708, inlink=0x2e61f00, in=0x2fa7e00) at libavfilter/framesync.c:312#4  0x000000000066e5c3 in ff_dualinput_filter_frame (s=0x2e60708, inlink=0x2e61f00, in=0x2fa7e00) at libavfilter/dualinput.c:79#5  0x0000000000541d9e in filter_frame (inlink=0x2e61f00, inpicref=0x2fa7e00) at libavfilter/vf_overlay.c:805#6  0x0000000000458992 in ff_filter_frame_framed (link=0x2e61f00, frame=0x2fa7e00) at libavfilter/avfilter.c:1116#7  0x0000000000458ff8 in ff_filter_frame_to_filter (link=0x2e61f00) at libavfilter/avfilter.c:1264#8  0x00000000004591f0 in ff_filter_activate_default (filter=0x2b6d7a0) at libavfilter/avfilter.c:1315#9  0x0000000000459350 in ff_filter_activate (filter=0x2b6d7a0) at libavfilter/avfilter.c:1476#10 0x000000000045db83 in ff_filter_graph_run_once (graph=0x29d6020) at libavfilter/avfiltergraph.c:1449#11 0x000000000045dd70 in get_frame_internal (ctx=0x2e612e0, frame=0x0, flags=1, samples=0) at libavfilter/buffersink.c:110#12 0x000000000045ddcd in av_buffersink_get_frame_flags (ctx=0x2e612e0, frame=0x0, flags=1) at libavfilter/buffersink.c:121#13 0x000000000045d916 in avfilter_graph_request_oldest (graph=0x29d6020) at libavfilter/avfiltergraph.c:1402#14 0x000000000042f25a in transcode_from_filter (graph=0x25154c0, best_ist=0x7fffffffdcf8) at ffmpeg.c:4455#15 0x000000000042f511 in transcode_step () at ffmpeg.c:4521#16 0x000000000042f789 in transcode () at ffmpeg.c:4597#17 0x000000000042fe75 in main (argc=8, argv=0x7fffffffdf88) at ffmpeg.c:4803

可以看出，17~6都是解码器完成的，filter只完成5~1即可

framesync

ffmpeg里面多路输入都要用framesync来同步
每一个输入在初始化过滤器的时候会有一个权重
权重最大的会触发on_event
on_event是一个callback函数
dualinput不过是利用framesync重写了一遍

filter里面相关函数

filter_frame

static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref){    OverlayContext *s = inlink->dst->priv;    //注意这里已经执行过config_input    av_log(inlink->dst, AV_LOG_DEBUG, "Incoming frame (time:%s) from link #%d\n", av_ts2timestr(inpicref->pts, &inlink->time_base), FF_INLINK_IDX(inlink));    return ff_dualinput_filter_frame(&s->dinput, inlink, inpicref);}

ff_dualinput_filter_frame

这个函数在dualinput.c里面

int ff_dualinput_filter_frame(FFDualInputContext *s,                                   AVFilterLink *inlink, AVFrame *in){    return ff_framesync_filter_frame(&s->fs, inlink, in);}

直接return ff_framesync_filter_frame

ff_framesync_filter_frame

位于framesync.h里面

int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink,                              AVFrame *in);/** * Request a frame on the filter output. * * This function can be the complete implementation of all filter_frame * methods of a filter using framesync if it has only one output. */

如果多路输入一个输出需要这个函数充当filter_frame()的角色
其实现为

int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink,                              AVFrame *in){    int ret;    if ((ret = ff_framesync_process_frame(fs, 1)) < 0)        return ret;    if ((ret = ff_framesync_add_frame(fs, FF_INLINK_IDX(inlink), in)) < 0)        return ret;    if ((ret = ff_framesync_process_frame(fs, 0)) < 0)        return ret;    return 0;}

这个函数分别执行三个函数，小于0则退出
在vf_overlay.c里面三个都执行了，do_blend是在第三次执行的时候被调用的

ff_framesync_process_frame

ff_framesync_process_frame的定义

int ff_framesync_process_frame(FFFrameSync *fs, unsigned all){    int ret, count = 0;    av_assert0(fs->on_event);    while (1) {        ff_framesync_next(fs);        if (fs->eof || !fs->frame_ready)            break;        if ((ret = fs->on_event(fs)) < 0)        //在这里调用process_frame，具体在下面说            return ret;        ff_framesync_drop(fs);        count++;        if (!all)            break;    }    if (!count && fs->eof)        return AVERROR_EOF;    return count;}

注意 FFFrameSync里面有

typedef struct FFFrameSync {    ......    int (*on_event)(struct FFFrameSync *fs);    /**     * Opaque pointer, not used by the API     */    ......} FFFrameSync;

dualinput.c的代码

static int process_frame(FFFrameSync *fs){    ......    if (secondpic && !ctx->is_disabled)        mainpic = s->process(ctx, mainpic, secondpic);    ret = ff_filter_frame(ctx->outputs[0], mainpic)}

int ff_dualinput_init(AVFilterContext *ctx, FFDualInputContext *s){    ......    s->fs.on_event = process_frame;    ......

ff_dualinput_uninit

对dinput进行初始化，在这里存放不同视频的帧数据

void ff_dualinput_uninit(FFDualInputContext *s){    ff_framesync_uninit(&s->fs);}

void ff_framesync_uninit(FFFrameSync *fs);/** * Add a frame to an input * * Typically called from the filter_frame() method. * * @param fs     frame sync structure * @param in     index of the input * @param frame  input frame, or NULL for EOF */

总结

对于双路输入，自己写过滤器的话，

1. 在filtercontext里面要包含一个FFDualInputContext的结构体
2. uninit进行初始化
3. 定义process回调函数
4. 在filter_frame里面调用ff_dualinput_filter_frame。