fftools/ffmpeg: propagate decoded_side_data from decoded streams to the filterchain

Global side data as exported by a decoder may no longer apply if a filter in
the chain altered the frames in some form, like changing color, dimensions,
or channel layout information.
After this change, any such changes in side data will be taken into account by
the encoder futher in the process.

Signed-off-by: James Almer <jamrial@gmail.com>

fftools/ffmpeg_dec.c

@@ -788,6 +788,11 @@ static int packet_decode(DecoderPriv *dp, AVPacket *pkt, AVFrame *frame)
 
         frame->time_base = dec->pkt_timebase;
 
+        ret = clone_side_data(&frame->side_data, &frame->nb_side_data,
+                              dec->decoded_side_data, dec->nb_decoded_side_data, 0);
+        if (ret < 0)
+            return ret;
+
         if (dec->codec_type == AVMEDIA_TYPE_AUDIO) {
             dp->dec.samples_decoded += frame->nb_samples;
 
@@ -1638,6 +1643,11 @@ static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts,
             param_out->color_range          = dp->dec_ctx->color_range;
         }
 
+        av_frame_side_data_free(&param_out->side_data, &param_out->nb_side_data);
+        ret = clone_side_data(&param_out->side_data, &param_out->nb_side_data,
+                              dp->dec_ctx->decoded_side_data, dp->dec_ctx->nb_decoded_side_data, 0);
+        if (ret < 0)
+            return ret;
         param_out->time_base = dp->dec_ctx->pkt_timebase;
     }
 

fftools/ffmpeg_filter.c

@@ -139,6 +139,9 @@ typedef struct InputFilterPriv {
 
     AVRational          time_base;
 
+    AVFrameSideData   **side_data;
+    int                 nb_side_data;
+
     AVFifo             *frame_queue;
 
     AVBufferRef        *hw_frames_ctx;
@@ -205,6 +208,9 @@ typedef struct OutputFilterPriv {
     enum AVColorSpace       color_space;
     enum AVColorRange       color_range;
 
+    AVFrameSideData       **side_data;
+    int                     nb_side_data;
+
     // time base in which the output is sent to our downstream
     // does not need to match the filtersink's timebase
     AVRational              tb_out;
@@ -1009,6 +1015,7 @@ void fg_free(FilterGraph **pfg)
         av_buffer_unref(&ifp->hw_frames_ctx);
         av_freep(&ifp->linklabel);
         av_freep(&ifp->opts.name);
+        av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data);
         av_freep(&ifilter->name);
         av_freep(&fg->inputs[j]);
     }
@@ -1026,6 +1033,7 @@ void fg_free(FilterGraph **pfg)
         av_freep(&ofilter->apad);
         av_freep(&ofp->name);
         av_channel_layout_uninit(&ofp->ch_layout);
+        av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data);
         av_freep(&fg->outputs[j]);
     }
     av_freep(&fg->outputs);
@@ -1725,6 +1733,9 @@ static int configure_input_video_filter(FilterGraph *fg, AVFilterGraph *graph,
     par->color_space         = ifp->color_space;
     par->color_range         = ifp->color_range;
     par->hw_frames_ctx       = ifp->hw_frames_ctx;
+    par->side_data           = ifp->side_data;
+    par->nb_side_data        = ifp->nb_side_data;
+
     ret = av_buffersrc_parameters_set(ifp->filter, par);
     if (ret < 0)
         goto fail;
@@ -1809,6 +1820,7 @@ static int configure_input_audio_filter(FilterGraph *fg, AVFilterGraph *graph,
 {
     InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
     AVFilterContext *last_filter;
+    AVBufferSrcParameters *par;
     const AVFilter *abuffer_filt = avfilter_get_by_name("abuffer");
     AVBPrint args;
     char name[255];
@@ -1831,6 +1843,15 @@ static int configure_input_audio_filter(FilterGraph *fg, AVFilterGraph *graph,
                                             name, args.str, NULL,
                                             graph)) < 0)
         return ret;
+    par = av_buffersrc_parameters_alloc();
+    if (!par)
+        return AVERROR(ENOMEM);
+    par->side_data     = ifp->side_data;
+    par->nb_side_data  = ifp->nb_side_data;
+    ret = av_buffersrc_parameters_set(ifp->filter, par);
+    av_free(par);
+    if (ret < 0)
+        return ret;
     last_filter = ifp->filter;
 
     snprintf(name, sizeof(name), "trim for input stream %s", ifp->opts.name);
@@ -1970,6 +1991,8 @@ static int configure_filtergraph(FilterGraph *fg, FilterGraphThread *fgt)
     /* limit the lists of allowed formats to the ones selected, to
      * make sure they stay the same if the filtergraph is reconfigured later */
     for (int i = 0; i < fg->nb_outputs; i++) {
+        const AVFrameSideData *const *sd;
+        int nb_sd;
         OutputFilter *ofilter = fg->outputs[i];
         OutputFilterPriv *ofp = ofp_from_ofilter(ofilter);
         AVFilterContext *sink = ofp->filter;
@@ -1998,6 +2021,17 @@ static int configure_filtergraph(FilterGraph *fg, FilterGraphThread *fgt)
         ret = av_buffersink_get_ch_layout(sink, &ofp->ch_layout);
         if (ret < 0)
             goto fail;
+        av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data);
+        sd = av_buffersink_get_side_data(sink, &nb_sd);
+        if (nb_sd)
+            for (int j = 0; j < nb_sd; j++) {
+                int ret = av_frame_side_data_clone(&ofp->side_data, &ofp->nb_side_data,
+                                                   sd[j], 0);
+                if (ret < 0) {
+                    av_frame_side_data_free(&ofp->side_data, &ofp->nb_side_data);
+                    goto fail;
+                }
+            }
     }
 
     for (int i = 0; i < fg->nb_inputs; i++) {
@@ -2066,6 +2100,20 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
     if (ret < 0)
         return ret;
 
+    av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data);
+    for (int i = 0; i < frame->nb_side_data; i++) {
+        const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type);
+
+        if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL))
+            continue;
+
+        ret = av_frame_side_data_clone(&ifp->side_data,
+                                       &ifp->nb_side_data,
+                                       frame->side_data[i], 0);
+        if (ret < 0)
+            return ret;
+    }
+
     sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
     if (sd)
         memcpy(ifp->displaymatrix, sd->data, sizeof(ifp->displaymatrix));
@@ -2396,6 +2444,11 @@ static int close_output(OutputFilterPriv *ofp, FilterGraphThread *fgt)
             if (ret < 0)
                 return ret;
         }
+        av_frame_side_data_free(&frame->side_data, &frame->nb_side_data);
+        ret = clone_side_data(&frame->side_data, &frame->nb_side_data,
+                              ofp->side_data, ofp->nb_side_data, 0);
+        if (ret < 0)
+            return ret;
 
         fd = frame_data(frame);
         if (!fd)
@@ -2748,6 +2801,13 @@ static int send_eof(FilterGraphThread *fgt, InputFilter *ifilter,
             if (ret < 0)
                 return ret;
 
+            av_frame_side_data_free(&ifp->side_data, &ifp->nb_side_data);
+            ret = clone_side_data(&ifp->side_data, &ifp->nb_side_data,
+                                  ifp->opts.fallback->side_data,
+                                  ifp->opts.fallback->nb_side_data, 0);
+            if (ret < 0)
+                return ret;
+
             if (ifilter_has_all_input_formats(ifilter->graph)) {
                 ret = configure_filtergraph(ifilter->graph, fgt);
                 if (ret < 0) {

fftools/ffmpeg_utils.h

@@ -44,4 +44,20 @@ static inline int err_merge(int err0, int err1)
     return (err0 < 0) ? err0 : FFMIN(err1, 0);
 }
 
+/**
+ * Wrapper calling av_frame_side_data_clone() in a loop for all source entries.
+ * It does not clear dst beforehand. */
+static inline int clone_side_data(AVFrameSideData ***dst, int *nb_dst,
+                                  AVFrameSideData * const *src, int nb_src,
+                                  unsigned int flags)
+{
+    for (int i = 0; i < nb_src; i++) {
+        int ret = av_frame_side_data_clone(dst, nb_dst, src[i], flags);
+        if (ret < 0)
+            return ret;
+    }
+
+    return 0;
+}
+
 #endif // FFTOOLS_FFMPEG_UTILS_H