FFmpeg原始帧处理-滤镜API用法详解

 _______        _____________________        ________
|       |      |                     |      |        |
| input | ---> | simple filter graph | ---> | output |
|_______|      |_____________________|      |________|

 _________
|         |
| input 0 |                    __________
|_________|                   |          |
                _________    /| output 0 |
               |         |  / |__________|
 _________     | complex | /
|         |     |         |/
| input 1 |---->| filter  |
|_________|     |         |    __________
               /| graph   |   |          |
              / |         |   | output 1 |
 _________   /  |_________|    |__________|
|         | /
| input 2 |/
|_________|

                [main]
input --> split ---------------------> overlay --> output
            |                             ^
            |[tmp]                  [flip]|
            +-----> crop --> vflip -------+

/**
 * Filter definition. This defines the pads a filter contains, and all the
 * callback functions used to interact with the filter.
 */
typedef struct AVFilter {
    const char *name;
    const char *description;
    const AVFilterPad *inputs;
    const AVFilterPad *outputs;
    const AVClass *priv_class;
    int flags;
    
    // private API
    ......
} AVFilter;

/** An instance of a filter */
struct AVFilterContext {
    const AVClass *av_class;        ///< needed for av_log() and filters common options

    const AVFilter *filter;         ///< the AVFilter of which this is an instance

    char *name;                     ///< name of this filter instance

    AVFilterPad   *input_pads;      ///< array of input pads
    AVFilterLink **inputs;          ///< array of pointers to input links
    unsigned    nb_inputs;          ///< number of input pads

    AVFilterPad   *output_pads;     ///< array of output pads
    AVFilterLink **outputs;         ///< array of pointers to output links
    unsigned    nb_outputs;         ///< number of output pads

    void *priv;                     ///< private data for use by the filter

    struct AVFilterGraph *graph;    ///< filtergraph this filter belongs to

    ......
};

typedef struct AVFilterGraph {
    const AVClass *av_class;
    AVFilterContext **filters;
    unsigned nb_filters;

    ......
} AVFilterGraph;

/**
 * A link between two filters. This contains pointers to the source and
 * destination filters between which this link exists, and the indexes of
 * the pads involved. In addition, this link also contains the parameters
 * which have been negotiated and agreed upon between the filter, such as
 * image dimensions, format, etc.
 *
 * Applications must not normally access the link structure directly.
 * Use the buffersrc and buffersink API instead.
 * In the future, access to the header may be reserved for filters
 * implementation.
 */
struct AVFilterLink {
    AVFilterContext *src;       ///< source filter
    AVFilterPad *srcpad;        ///< output pad on the source filter

    AVFilterContext *dst;       ///< dest filter
    AVFilterPad *dstpad;        ///< input pad on the dest filter
    
    ......
}

/**
 * A linked-list of the inputs/outputs of the filter chain.
 *
 * This is mainly useful for avfilter_graph_parse() / avfilter_graph_parse2(),
 * where it is used to communicate open (unlinked) inputs and outputs from and
 * to the caller.
 * This struct specifies, per each not connected pad contained in the graph, the
 * filter context and the pad index required for establishing a link.
 */
typedef struct AVFilterInOut {
    /** unique name for this input/output in the list */
    char *name;

    /** filter context associated to this input/output */
    AVFilterContext *filter_ctx;

    /** index of the filt_ctx pad to use for linking */
    int pad_idx;

    /** next input/input in the list, NULL if this is the last */
    struct AVFilterInOut *next;
} AVFilterInOut;

/**
 * Create and add a filter instance into an existing graph.
 * The filter instance is created from the filter filt and inited
 * with the parameters args and opaque.
 *
 * In case of success put in *filt_ctx the pointer to the created
 * filter instance, otherwise set *filt_ctx to NULL.
 *
 * @param name the instance name to give to the created filter instance
 * @param graph_ctx the filter graph
 * @return a negative AVERROR error code in case of failure, a non
 * negative value otherwise
 */
int avfilter_graph_create_filter(AVFilterContext **filt_ctx, const AVFilter *filt,
                                 const char *name, const char *args, void *opaque,
                                 AVFilterGraph *graph_ctx);

/**
 * Add a graph described by a string to a graph.
 *
 * In the graph filters description, if the input label of the first
 * filter is not specified, "in" is assumed; if the output label of
 * the last filter is not specified, "out" is assumed.
 *
 * @param graph   the filter graph where to link the parsed graph context
 * @param filters string to be parsed
 * @param inputs  pointer to a linked list to the inputs of the graph, may be NULL.
 *                If non-NULL, *inputs is updated to contain the list of open inputs
 *                after the parsing, should be freed with avfilter_inout_free().
 * @param outputs pointer to a linked list to the outputs of the graph, may be NULL.
 *                If non-NULL, *outputs is updated to contain the list of open outputs
 *                after the parsing, should be freed with avfilter_inout_free().
 * @return non negative on success, a negative AVERROR code on error
 */
int avfilter_graph_parse_ptr(AVFilterGraph *graph, const char *filters,
                             AVFilterInOut **inputs, AVFilterInOut **outputs,
                             void *log_ctx);

/**
 * Check validity and configure all the links and formats in the graph.
 *
 * @param graphctx the filter graph
 * @param log_ctx context used for logging
 * @return >= 0 in case of success, a negative AVERROR code otherwise
 */
int avfilter_graph_config(AVFilterGraph *graphctx, void *log_ctx);

/**
 * Add a frame to the buffer source.
 *
 * By default, if the frame is reference-counted, this function will take
 * ownership of the reference(s) and reset the frame. This can be controlled
 * using the flags.
 *
 * If this function returns an error, the input frame is not touched.
 *
 * @param buffer_src  pointer to a buffer source context
 * @param frame       a frame, or NULL to mark EOF
 * @param flags       a combination of AV_BUFFERSRC_FLAG_*
 * @return            >= 0 in case of success, a negative AVERROR code
 *                    in case of failure
 */
av_warn_unused_result
int av_buffersrc_add_frame_flags(AVFilterContext *buffer_src,
                                 AVFrame *frame, int flags);

/**
 * Get a frame with filtered data from sink and put it in frame.
 *
 * @param ctx pointer to a context of a buffersink or abuffersink AVFilter.
 * @param frame pointer to an allocated frame that will be filled with data.
 *              The data must be freed using av_frame_unref() / av_frame_free()
 *
 * @return
 *         - >= 0 if a frame was successfully returned.
 *         - AVERROR(EAGAIN) if no frames are available at this point; more
 *           input frames must be added to the filtergraph to get more output.
 *         - AVERROR_EOF if there will be no more output frames on this sink.
 *         - A different negative AVERROR code in other failure cases.
 */
int av_buffersink_get_frame(AVFilterContext *ctx, AVFrame *frame);

1. init_filters()                   // 配置生成可用的滤镜图，由用户编写  
2. av_buffersrc_add_frame_flags()   // 向滤镜图提供输入帧，API函数  
3. av_buffersink_get_frame()        // 从滤镜图取出处理后的输出帧，API函数  

// 功能：创建配置一个滤镜图，在后续滤镜处理中，可以往此滤镜图输入数据并从滤镜图获得输出数据
// filters_descr：输入参数，形如“transpose=cclock,pad=iw+80:ih:40”
// @vfmt：输入参数，描述提供给待生成滤镜图的视频帧和格式
// @fctx：输出参数，返回生成滤镜图的信息，供调用者使用
int init_filters(const char *filters_descr, const input_vfmt_t *vfmt, filter_ctx_t *fctx)
{
    int ret = 0;

    // 1. 配置滤镜图输入端和输出端 
    fctx->filter_graph = avfilter_graph_alloc();
    if (!fctx->filter_graph)
    {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    char args[512];
    char *p_args = NULL;
    if (vfmt != NULL)
    {
        /* buffer video source: the decoded frames from the decoder will be inserted here. */
        // args是buffersrc滤镜的参数
        snprintf(args, sizeof(args),
                 "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
                 vfmt->width, vfmt->height, vfmt->pix_fmt, 
                 vfmt->time_base.num, vfmt->time_base.den, 
                 vfmt->sar.num, vfmt->sar.den);
        p_args = args;
    }
    ret = avfilter_graph_create_filter(&fctx->bufsrc_ctx, bufsrc, "in",
                                       p_args, NULL, fctx->filter_graph);
    if (ret < 0)
    {
        av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sourcen");
        goto end;
    }

    const AVFilter *bufsink = avfilter_get_by_name("buffersink");
    ret = avfilter_graph_create_filter(&fctx->bufsink_ctx, bufsink, "out",
                                       NULL, NULL, fctx->filter_graph);
    if (ret < 0)
    {
        av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sinkn");
        goto end;
    }

#if 0   // 因为后面显示视频帧时有sws_scale()进行图像格式转换，故此处不设置滤镜输出格式也可
    enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUYV422, AV_PIX_FMT_NONE };
    // 设置输出像素格式为pix_fmts[]中指定的格式(如果要用SDL显示，则这些格式应是SDL支持格式)
    ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
                              AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
    if (ret < 0) {
        av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel formatn");
        goto end;
    }
#endif
    // 1. end


    // 2. 将filters_descr描述的滤镜图添加到filter_graph滤镜图中
    AVFilterInOut *outputs = avfilter_inout_alloc();
    outputs->name       = av_strdup("in");
    outputs->filter_ctx = fctx->bufsrc_ctx;
    outputs->pad_idx    = 0;
    outputs->next       = NULL;

    AVFilterInOut *inputs  = avfilter_inout_alloc();
    inputs->name       = av_strdup("out");
    inputs->filter_ctx = fctx->bufsink_ctx;
    inputs->pad_idx    = 0;
    inputs->next       = NULL;

    ret = avfilter_graph_parse_ptr(fctx->filter_graph, filters_descr,
                                   &inputs, &outputs, NULL);
    if (ret < 0)
    {
        goto end;
    }
    // 2. end

    // 3. 配置filtergraph滤镜图，建立滤镜间的连接
    ret = avfilter_graph_config(fctx->filter_graph, NULL);
    if (ret < 0)
    {
        goto end;
    }
    // 3. end

end:
    avfilter_inout_free(&inputs);
    avfilter_inout_free(&outputs);

    return ret;
}

typedef struct {
    int width;
    int height;
    enum AVPixelFormat pix_fmt;
    AVRational time_base;
    AVRational sar;
    AVRational frame_rate;
}   input_vfmt_t;

typedef struct {
    AVFilterContext *bufsink_ctx;
    AVFilterContext *bufsrc_ctx;
    AVFilterGraph   *filter_graph;
}   filter_ctx_t;

$ ffmpeg -h filter=buffer
ffmpeg version 4.1 Copyright (c) 2000-2018 the FFmpeg developers
Filter buffer
  Buffer video frames, and make them accessible to the filterchain.
    Inputs:
        none (source filter)
    Outputs:
       #0: default (video)
buffer AVOptions:
  width             <int>        ..FV..... (from 0 to INT_MAX) (default 0)
  video_size        <image_size> ..FV.....
  height            <int>        ..FV..... (from 0 to INT_MAX) (default 0)
  pix_fmt           <pix_fmt>    ..FV..... (default none)
  sar               <rational>   ..FV..... sample aspect ratio (from 0 to DBL_MAX) (default 0/1)
  pixel_aspect      <rational>   ..FV..... sample aspect ratio (from 0 to DBL_MAX) (default 0/1)
  time_base         <rational>   ..FV..... (from 0 to DBL_MAX) (default 0/1)
  frame_rate        <rational>   ..FV..... (from 0 to DBL_MAX) (default 0/1)
  sws_param         <string>     ..FV.....

    char args[512];
    char *p_args = NULL;
    if (vfmt != NULL)
    {
        // args是buffersrc滤镜的参数
        snprintf(args, sizeof(args),
                 "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
                 vfmt->width, vfmt->height, vfmt->pix_fmt, 
                 vfmt->time_base.num, vfmt->time_base.den, 
                 vfmt->sar.num, vfmt->sar.den);
        p_args = args;
    }
    // buffer滤镜：缓冲视频帧，作为滤镜图的输入
    const AVFilter *bufsrc  = avfilter_get_by_name("buffer");
    // 为buffersrc滤镜创建滤镜实例buffersrc_ctx，命名为"in"
    // 将新创建的滤镜实例buffersrc_ctx添加到滤镜图filter_graph中
    ret = avfilter_graph_create_filter(&fctx->bufsrc_ctx, bufsrc, "in",
                                       p_args, NULL, fctx->filter_graph);

$  ffmpeg -h filter=buffersink
ffmpeg version 4.1 Copyright (c) 2000-2018 the FFmpeg developers
Filter buffersink
  Buffer video frames, and make them available to the end of the filter graph.
    Inputs:
       #0: default (video)
    Outputs:
        none (sink filter)
buffersink AVOptions:
  pix_fmts          <binary>     ..FV..... set the supported pixel formats

    // buffersink滤镜：缓冲视频帧，作为滤镜图的输出
    const AVFilter *bufsink = avfilter_get_by_name("buffersink");
    // 为buffersink滤镜创建滤镜实例buffersink_ctx，命名为"out"
    // 将新创建的滤镜实例buffersink_ctx添加到滤镜图filter_graph中
    ret = avfilter_graph_create_filter(&fctx->bufsink_ctx, bufsink, "out",
                                       NULL, NULL, fctx->filter_graph);

#if 0   // 因为后面显示视频帧时有sws_scale()进行图像格式转换，故此处不设置滤镜输出格式也可
    enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUYV422, AV_PIX_FMT_NONE };
    // 设置输出像素格式为pix_fmts[]中指定的格式(如果要用SDL显示，则这些格式应是SDL支持格式)
    ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
                              AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
#endif

    // 设置滤镜图的端点，将filters_descr描述的滤镜图连接到此滤镜图
    // 两个滤镜图的连接是通过端点(AVFilterInOut)连接完成的
    // 端点数据结构AVFilterInOut主要用于avfilter_graph_parse()系列函数

    // outputs变量意指buffersrc_ctx滤镜的输出引脚(output pad)
    // src缓冲区(buffersrc_ctx滤镜)的输出必须连到filters_descr中第一个
    // 滤镜的输入；filters_descr中第一个滤镜的输入标号未指定，故默认为
    // "in"，此处将buffersrc_ctx的输出标号也设为"in"，就实现了同标号相连
    AVFilterInOut *outputs = avfilter_inout_alloc();
    outputs->name       = av_strdup("in");
    outputs->filter_ctx = fctx->bufsrc_ctx;
    outputs->pad_idx    = 0;
    outputs->next       = NULL;

    // inputs变量意指buffersink_ctx滤镜的输入引脚(input pad)
    // sink缓冲区(buffersink_ctx滤镜)的输入必须连到filters_descr中最后
    // 一个滤镜的输出；filters_descr中最后一个滤镜的输出标号未指定，故
    // 默认为"out"，此处将buffersink_ctx的输出标号也设为"out"，就实现了
    // 同标号相连
    AVFilterInOut *inputs  = avfilter_inout_alloc();
    inputs->name       = av_strdup("out");
    inputs->filter_ctx = fctx->bufsink_ctx;
    inputs->pad_idx    = 0;
    inputs->next       = NULL;

    // 将filters_descr描述的滤镜图添加到filter_graph滤镜图中
    // 调用前：filter_graph包含两个滤镜buffersrc_ctx和buffersink_ctx
    // 调用后：filters_descr描述的滤镜图插入到filter_graph中，buffersrc_ctx连接到filters_descr
    //         的输入，filters_descr的输出连接到buffersink_ctx，filters_descr只进行了解析而不
    //         建立内部滤镜间的连接。filters_desc与filter_graph间的连接是利用AVFilterInOut inputs
    //         和AVFilterInOut outputs连接起来的，AVFilterInOut是一个链表，最终可用的连在一起的
    //         滤镜链/滤镜图就是通过这个链表串在一起的。
    ret = avfilter_graph_parse_ptr(fctx->filter_graph, filters_descr,
                                   &inputs, &outputs, NULL);

svn checkout https://github.com/leichn/exercises/trunk/source/ffmpeg/ffmpeg_vfilter/

vfilter_filesrc.c   用于示例2：输入源为视频文件，经滤镜处理后播放
vfilter_testsrc.c   用于示例3：输入源为测试图，经滤镜处理后播放
video_filter.c      滤镜处理功能
video_play.c        视频播放功能
Makefile

// @filter [i]  产生测试图案的filter
// @vfmt   [i]  @filter的参数
// @fctx   [o]  用户定义的数据类型，输出供调用者使用
static int open_testsrc(const char *filter, const input_vfmt_t *vfmt, filter_ctx_t *fctx)
{
    int ret = 0;

    // 分配一个滤镜图filter_graph
    fctx->filter_graph = avfilter_graph_alloc();
    if (!fctx->filter_graph)
    {
        return AVERROR(ENOMEM);
    }

    // source滤镜：合法值有"testsrc"/"smptebars"/"color"/...
    const AVFilter *bufsrc  = avfilter_get_by_name(filter);
    // 为buffersrc滤镜创建滤镜实例buffersrc_ctx，命名为"in"
    // 将新创建的滤镜实例buffersrc_ctx添加到滤镜图filter_graph中
    ret = avfilter_graph_create_filter(&fctx->bufsrc_ctx, bufsrc, "in",
                                       NULL, NULL, fctx->filter_graph);
    if (ret < 0)
    {
        av_log(NULL, AV_LOG_ERROR, "Cannot create filter testsrcn");
        goto end;
    }

    // "buffersink"滤镜：缓冲视频帧，作为滤镜图的输出
    const AVFilter *bufsink = avfilter_get_by_name("buffersink");
    /* buffer video sink: to terminate the filter chain. */
    // 为buffersink滤镜创建滤镜实例buffersink_ctx，命名为"out"
    // 将新创建的滤镜实例buffersink_ctx添加到滤镜图filter_graph中
    ret = avfilter_graph_create_filter(&fctx->bufsink_ctx, bufsink, "out",
                                       NULL, NULL, fctx->filter_graph);
    if (ret < 0)
    {
        av_log(NULL, AV_LOG_ERROR, "Cannot create filter buffersinkn");
        goto end;
    }

    if ((ret = avfilter_link(fctx->bufsrc_ctx, 0, fctx->bufsink_ctx, 0)) < 0)
    {
        goto end;
    }


    // 验证有效性并配置filtergraph中所有连接和格式
    ret = avfilter_graph_config(fctx->filter_graph, NULL);
    if (ret < 0)
    {
        goto end;
    }

    vfmt->pix_fmt = av_buffersink_get_format(fctx->bufsink_ctx);
    vfmt->width = av_buffersink_get_w(fctx->bufsink_ctx);
    vfmt->height = av_buffersink_get_h(fctx->bufsink_ctx);
    vfmt->sar = av_buffersink_get_sample_aspect_ratio(fctx->bufsink_ctx);
    vfmt->time_base = av_buffersink_get_time_base(fctx->bufsink_ctx);
    vfmt->frame_rate = av_buffersink_get_frame_rate(fctx->bufsink_ctx);

    av_log(NULL, AV_LOG_INFO, "probe video format: "
           "%dx%d, pix_fmt %d, SAR %d/%d, tb %d/%d, rate %d/%dn",
           vfmt->width, vfmt->height, vfmt->pix_fmt,
           vfmt->sar.num, vfmt->sar.den,
           vfmt->time_base.num, vfmt->time_base.den,
           vfmt->frame_rate.num, vfmt->frame_rate.den);

    return 0;

end:
    avfilter_graph_free(&fctx->filter_graph);
    return ret;
}

ffplay -f lavfi -i testsrc

ffplay -f lavfi -i testsrc -vf transpose=cclock,pad=iw+80:ih:40

./vf_test testsrc -vf transpose=cclock,pad=iw+80:ih:40

ffplay -f lavfi -i smptebars

ffplay -f lavfi -i smptebars -vf transpose=cclock,pad=iw+80:ih:40

./vf_test smptebars -vf transpose=cclock,pad=iw+80:ih:40

think@linux-1phi:~> ffprobe -f lavfi -i testsrc
ffprobe version 4.1 Copyright (c) 2007-2018 the FFmpeg developers
Input #0, lavfi, from 'testsrc':
  Duration: N/A, start: 0.000000, bitrate: N/A
    Stream #0:0: Video: rawvideo (RGB[24] / 0x18424752), rgb24, 320x240 [SAR 1:1 DAR 4:3], 25 tbr, 25 tbn, 25 tbc