avfilter: add colormap video filter

Changelog

@@ -11,6 +11,7 @@ version 5.1:
 - avsynctest source filter
 - feedback video filter
 - pixelize video filter
+- colormap video filter
 
 
 version 5.0:

doc/filters.texi

@@ -8944,6 +8944,39 @@ colorlevels=romin=0.5:gomin=0.5:bomin=0.5
 
 This filter supports the all above options as @ref{commands}.
 
+@section colormap
+Apply custom color maps to video stream.
+
+This filter needs three input video streams.
+First stream is video stream that is going to be filtered out.
+Second and third video stream specify color patches for source
+color to target color mapping.
+
+The filter accepts the following options:
+
+@table @option
+@item patch_size
+Set the source and target video stream patch size in pixels.
+
+@item nb_patches
+Set the max number of used patches from source and target video stream.
+
+@item type
+Set the adjustments used for target colors. Can be @code{relative} or @code{absolute}.
+Defaults is @code{absolute}.
+
+@item kernel
+Set the kernel used to measure color differences between mapped colors.
+
+The accepted values are:
+@table @samp
+@item euclidean
+@item weuclidean
+@end table
+
+Default is @code{euclidean}.
+@end table
+
 @section colormatrix
 
 Convert color matrix.

libavfilter/Makefile

@@ -218,6 +218,7 @@ OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER)        += vf_colorkey_opencl.o opencl.o \
                                                 opencl/colorkey.o
 OBJS-$(CONFIG_COLORHOLD_FILTER)              += vf_colorkey.o
 OBJS-$(CONFIG_COLORLEVELS_FILTER)            += vf_colorlevels.o
+OBJS-$(CONFIG_COLORMAP_FILTER)               += vf_colormap.o
 OBJS-$(CONFIG_COLORMATRIX_FILTER)            += vf_colormatrix.o
 OBJS-$(CONFIG_COLORSPACE_FILTER)             += vf_colorspace.o colorspace.o colorspacedsp.o
 OBJS-$(CONFIG_COLORTEMPERATURE_FILTER)       += vf_colortemperature.o

libavfilter/allfilters.c

@@ -204,6 +204,7 @@ extern const AVFilter ff_vf_colorkey;
 extern const AVFilter ff_vf_colorkey_opencl;
 extern const AVFilter ff_vf_colorhold;
 extern const AVFilter ff_vf_colorlevels;
+extern const AVFilter ff_vf_colormap;
 extern const AVFilter ff_vf_colormatrix;
 extern const AVFilter ff_vf_colorspace;
 extern const AVFilter ff_vf_colortemperature;

libavfilter/version.h

@@ -31,7 +31,7 @@
 
 #include "version_major.h"
 
-#define LIBAVFILTER_VERSION_MINOR  33
+#define LIBAVFILTER_VERSION_MINOR  34
 #define LIBAVFILTER_VERSION_MICRO 100
 
 

libavfilter/vf_colormap.c

@@ -0,0 +1,582 @@
+/*
+ * Copyright (c) 2022 Paul B Mahol
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Compute a look-up table from map of colors.
+ */
+
+#include "config_components.h"
+
+#include "libavutil/attributes.h"
+#include "libavutil/avstring.h"
+#include "libavutil/common.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "avfilter.h"
+#include "formats.h"
+#include "internal.h"
+#include "framesync.h"
+#include "video.h"
+
+#define MAX_SIZE 64
+
+enum KernelType {
+    EUCLIDEAN,
+    WEUCLIDEAN,
+    NB_KERNELS,
+};
+
+typedef struct ColorMapContext {
+    const AVClass *class;
+    int w, h;
+    int size;
+    int nb_maps;
+    int changed[2];
+
+    float source[MAX_SIZE][4];
+    float ttarget[MAX_SIZE][4];
+    float target[MAX_SIZE][4];
+    float icoeff[4][4];
+    float coeff[MAX_SIZE][4];
+
+    int target_type;
+    int kernel_type;
+    float (*kernel)(const float *x, const float *y);
+
+    FFFrameSync fs;
+} ColorMapContext;
+
+#define OFFSET(x) offsetof(ColorMapContext, x)
+#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
+
+static const AVOption colormap_options[] = {
+    { "patch_size", "set patch size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "64x64"}, 0, 0, FLAGS },
+    { "nb_patches", "set number of patches", OFFSET(size), AV_OPT_TYPE_INT, {.i64 = 8}, 1, MAX_SIZE, FLAGS },
+    { "type", "set the target type used",  OFFSET(target_type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, FLAGS, "type" },
+    {   "relative", "the target colors are relative", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 1, FLAGS, "type" },
+    {   "absolute", "the target colors are absolute", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 1, FLAGS, "type" },
+    { "kernel", "set the kernel used for measuring color difference",  OFFSET(kernel_type), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_KERNELS-1, FLAGS, "kernel" },
+    {   "euclidean",   "square root of sum of squared differences",         0, AV_OPT_TYPE_CONST, {.i64=EUCLIDEAN},   0, 0, FLAGS, "kernel" },
+    {   "weuclidean",  "weighted square root of sum of squared differences",0, AV_OPT_TYPE_CONST, {.i64=WEUCLIDEAN},  0, 0, FLAGS, "kernel" },
+    { NULL }
+};
+
+static int gauss_make_triangular(double *A, int *p, int n)
+{
+    p[n - 1] = n - 1;
+    for (int k = 0; k < n; k++) {
+        double t1;
+        int m = k;
+
+        for (int i = k + 1; i < n; i++)
+            if (fabs(A[k + n * i]) > fabs(A[k + n * m]))
+                m = i;
+        p[k] = m;
+        t1 = A[k + n * m];
+        A[k + n * m] = A[k + n * k];
+        A[k + n * k] = t1;
+        if (t1 != 0) {
+            for (int i = k + 1; i < n; i++)
+                A[k + n * i] /= -t1;
+            if (k != m)
+                for (int i = k + 1; i < n; i++) {
+                    double t2 = A[i + n * m];
+                    A[i + n * m] = A[i + n * k];
+                    A[i + n * k] = t2;
+                }
+            for (int j = k + 1; j < n; j++)
+                for (int i = k + 1; i < n; i++)
+                    A[i + n * j] += A[k + j * n] * A[i + k * n];
+        } else {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+static void gauss_solve_triangular(const double *A, const int *p, double *b, int n)
+{
+    for(int k = 0; k < n - 1; k++) {
+        int m = p[k];
+        double t = b[m];
+        b[m] = b[k];
+        b[k] = t;
+        for (int i = k + 1; i < n; i++)
+            b[i] += A[k + n * i] * t;
+    }
+
+    for(int k = n - 1; k > 0; k--) {
+        b[k] /= A[k + n * k];
+        double t = b[k];
+        for (int i = 0; i < k; i++)
+            b[i] -= A[k + n * i] * t;
+    }
+
+    b[0] /= A[0 + 0 * n];
+}
+
+static int gauss_solve(double *A, double *b, int n)
+{
+    int *p = av_calloc(n, sizeof(*p));
+
+    if (!p)
+        return 1;
+
+    if (!gauss_make_triangular(A, p, n)) {
+        av_freep(&p);
+        return 1;
+    }
+
+    gauss_solve_triangular(A, p, b, n);
+
+    av_freep(&p);
+
+    return 0;
+}
+
+#define P2(x) ((x)*(x))
+
+static float euclidean_kernel(const float *x, const float *y)
+{
+    const float d2 = P2(x[0]-y[0]) +
+                     P2(x[1]-y[1]) +
+                     P2(x[2]-y[2]);
+    return sqrtf(d2);
+}
+
+static float weuclidean_kernel(const float *x, const float *y)
+{
+    const float rm = (x[0] + y[0]) * 0.5f;
+    const float d2 = P2(x[0]-y[0]) * (2.f + rm) +
+                     P2(x[1]-y[1]) * 4.f +
+                     P2(x[2]-y[2]) * (3.f - rm);
+    return sqrtf(d2);
+}
+
+static void build_map(AVFilterContext *ctx)
+{
+    ColorMapContext *s = ctx->priv;
+
+    for (int j = 0; j < s->nb_maps; j++) {
+        s->target[j][0] = s->target_type == 0 ? s->source[j][0] + s->ttarget[j][0] : s->ttarget[j][0];
+        s->target[j][1] = s->target_type == 0 ? s->source[j][1] + s->ttarget[j][1] : s->ttarget[j][1];
+        s->target[j][2] = s->target_type == 0 ? s->source[j][2] + s->ttarget[j][2] : s->ttarget[j][2];
+    }
+
+    for (int c = 0; c < 3; c++) {
+        for (int j = 0; j < s->nb_maps; j++)
+            s->coeff[j][c] = 0.f;
+
+        for (int j = 0; j < 4; j++) {
+            s->icoeff[j][c] = 0;
+            s->icoeff[j][c] = 0;
+            s->icoeff[j][c] = 0;
+        }
+
+        s->icoeff[c+1][c] = 1.f;
+
+        switch (s->nb_maps) {
+        case 1:
+            {
+                float div = fabsf(s->source[0][c]) < 1e-6f ? 1e-6f : s->source[0][c];
+                s->icoeff[c][1+c] = s->target[0][c] / div;
+            }
+            break;
+        case 2:
+            {
+                double A[2 * 2] = { 1, s->source[0][c],
+                                    1, s->source[1][c] };
+                double b[2] = { s->target[0][c], s->target[1][c] };
+
+                if (gauss_solve(A, b, 2))
+                    continue;
+
+                s->icoeff[0  ][c] = b[0];
+                s->icoeff[1+c][c] = b[1];
+            }
+            break;
+        case 3:
+            {
+                const uint8_t idx[3][3] = {{ 0, 1, 2 },
+                                           { 1, 0, 2 },
+                                           { 2, 0, 1 }};
+                const uint8_t didx[3][4] = {{ 0, 1, 2, 2 },
+                                            { 0, 2, 1, 2 },
+                                            { 0, 2, 2, 1 }};
+                const int C0 = idx[c][0];
+                const int C1 = idx[c][1];
+                const int C2 = idx[c][2];
+                double A[3 * 3] = { 1, s->source[0][C0], s->source[0][C1] + s->source[0][C2],
+                                    1, s->source[1][C0], s->source[1][C1] + s->source[1][C2],
+                                    1, s->source[2][C0], s->source[2][C1] + s->source[2][C2] };
+                double b[3] = { s->target[0][c], s->target[1][c], s->target[2][c] };
+
+                if (gauss_solve(A, b, 3))
+                    continue;
+
+                s->icoeff[0][c] = b[didx[c][0]];
+                s->icoeff[1][c] = b[didx[c][1]];
+                s->icoeff[2][c] = b[didx[c][2]];
+                s->icoeff[3][c] = b[didx[c][3]];
+            }
+            break;
+        case 4:
+            {
+                double A[4 * 4] = { 1, s->source[0][0], s->source[0][1], s->source[0][2],
+                                    1, s->source[1][0], s->source[1][1], s->source[1][2],
+                                    1, s->source[2][0], s->source[2][1], s->source[2][2],
+                                    1, s->source[3][0], s->source[3][1], s->source[3][2] };
+                double b[4] = { s->target[0][c], s->target[1][c], s->target[2][c], s->target[3][c] };
+                int pivot[4];
+
+                if (!gauss_make_triangular(A, pivot, 4))
+                    continue;
+                gauss_solve_triangular(A, pivot, b, 4);
+
+                s->icoeff[0][c] = b[0];
+                s->icoeff[1][c] = b[1];
+                s->icoeff[2][c] = b[2];
+                s->icoeff[3][c] = b[3];
+            }
+            break;
+        default:
+            {
+                const int N = s->nb_maps;
+                const int N4 = N + 4;
+                double *A = av_calloc(sizeof(*A), N4 * N4);
+                double *b = av_calloc(sizeof(*b), N4);
+                int *pivot = NULL;
+
+                if (!A || !b)
+                    goto error;
+
+                for (int j = 0; j < N; j++)
+                    for (int i = j; i < N; i++)
+                        A[j*N4+i] = A[i*N4+j] = s->kernel(s->source[i], s->source[j]);
+
+                for (int i = 0; i < N; i++)
+                    A[i*N4+N+0] = A[(N+0)*N4+i] = 1;
+                for (int i = 0; i < N; i++)
+                    A[i*N4+N+1] = A[(N+1)*N4+i] = s->source[i][0];
+                for (int i = 0; i < N; i++)
+                    A[i*N4+N+2] = A[(N+2)*N4+i] = s->source[i][1];
+                for (int i = 0; i < N; i++)
+                    A[i*N4+N+3] = A[(N+3)*N4+i] = s->source[i][2];
+
+                for (int j = N; j < N4; j++)
+                    for (int i = N;i < N4; i++)
+                        A[j * N4 + i] = 0.;
+
+                pivot = av_calloc(N4, sizeof(*pivot));
+                if (!pivot)
+                    goto error;
+
+                if (gauss_make_triangular(A, pivot, N4)) {
+                    for (int i = 0; i < N; i++)
+                        b[i] = s->target[i][c];
+                    for (int i = N; i < N + 4; i++)
+                        b[i] = 0;
+
+                    gauss_solve_triangular(A, pivot, b, N4);
+
+                    for (int i = 0; i < N; i++)
+                        s->coeff[i][c] = b[i];
+
+                    for (int i = 0; i < 4; i++)
+                        s->icoeff[i][c] = b[N + i];
+                }
+error:
+                av_free(pivot);
+                av_free(b);
+                av_free(A);
+            }
+        }
+    }
+}
+
+typedef struct ThreadData {
+    AVFrame *in, *out;
+} ThreadData;
+
+static int colormap_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+    ColorMapContext *s = ctx->priv;
+    ThreadData *td = arg;
+    AVFrame *in = td->in;
+    AVFrame *out = td->out;
+    const int maps = s->nb_maps;
+    const int width = out->width;
+    const int height = out->height;
+    const int slice_start = (height * jobnr) / nb_jobs;
+    const int slice_end = (height * (jobnr + 1)) / nb_jobs;
+    const int sr_linesize = in->linesize[2] / 4;
+    const int dr_linesize = out->linesize[2] / 4;
+    const int sg_linesize = in->linesize[0] / 4;
+    const int dg_linesize = out->linesize[0] / 4;
+    const int sb_linesize = in->linesize[1] / 4;
+    const int db_linesize = out->linesize[1] / 4;
+    const float *sr = (float *)in->data[2] + slice_start * sr_linesize;
+    const float *sg = (float *)in->data[0] + slice_start * sg_linesize;
+    const float *sb = (float *)in->data[1] + slice_start * sb_linesize;
+    float *r = (float *)out->data[2] + slice_start * dr_linesize;
+    float *g = (float *)out->data[0] + slice_start * dg_linesize;
+    float *b = (float *)out->data[1] + slice_start * db_linesize;
+    float (*kernel)(const float *x, const float *y) = s->kernel;
+    const float *icoeff[4] = { s->icoeff[0], s->icoeff[1], s->icoeff[2], s->icoeff[3] };
+
+    for (int y = slice_start; y < slice_end; y++) {
+        for (int x = 0; x < width; x++) {
+            const float input[3] = { sr[x], sg[x], sb[x] };
+            float srv, sgv, sbv;
+            float rv, gv, bv;
+
+            srv = sr[x];
+            sgv = sg[x];
+            sbv = sb[x];
+
+            rv = icoeff[0][0];
+            gv = icoeff[0][1];
+            bv = icoeff[0][2];
+
+            rv += icoeff[1][0] * srv + icoeff[2][0] * sgv + icoeff[3][0] * sbv;
+            gv += icoeff[1][1] * srv + icoeff[2][1] * sgv + icoeff[3][1] * sbv;
+            bv += icoeff[1][2] * srv + icoeff[2][2] * sgv + icoeff[3][2] * sbv;
+
+            for (int z = 0; z < maps && maps > 4; z++) {
+                const float *coeff = s->coeff[z];
+                const float cr = coeff[0];
+                const float cg = coeff[1];
+                const float cb = coeff[2];
+                const float f = kernel(input, s->source[z]);
+
+                rv += f * cr;
+                gv += f * cg;
+                bv += f * cb;
+            }
+
+            r[x] = rv;
+            g[x] = gv;
+            b[x] = bv;
+        }
+
+        sg += sg_linesize;
+        g += dg_linesize;
+        sb += sb_linesize;
+        b += db_linesize;
+        sr += sr_linesize;
+        r += dr_linesize;
+    }
+
+    return 0;
+}
+
+static int import_map(AVFilterLink *inlink, AVFrame *in)
+{
+    AVFilterContext *ctx = inlink->dst;
+    ColorMapContext *s = ctx->priv;
+    const int is_target = FF_INLINK_IDX(inlink) > 1;
+    const int pw = s->w;
+    const int pw2 = s->w / 2;
+    const int ph = s->h;
+    const int ph2 = s->h / 2;
+    int changed = 0;
+    int idx;
+
+    for (int plane = 0; plane < 3; plane++) {
+        const int c = plane == 0 ? 1 : plane == 1 ? 2 : 0;
+
+        idx = 0;
+        for (int y = ph2; y < in->height && idx < MAX_SIZE; y += ph) {
+            const float *src = (const float *)(in->data[plane] + y * in->linesize[plane]);
+
+            for (int x = pw2; x < in->width && idx < MAX_SIZE; x += pw) {
+                float value = src[x];
+
+                if (is_target) {
+                    if (s->ttarget[idx][c] != value)
+                        changed = 1;
+                    s->ttarget[idx][c] = value;
+                } else {
+                    if (s->source[idx][c] != value)
+                        changed = 1;
+                    s->source[idx][c] = value;
+                }
+
+                idx++;
+            }
+        }
+    }
+
+    if (changed)
+        s->changed[is_target] = 1;
+    if (!is_target)
+        s->nb_maps = FFMIN(idx, s->size);
+
+    return 0;
+}
+
+static int process_frame(FFFrameSync *fs)
+{
+    AVFilterContext *ctx = fs->parent;
+    ColorMapContext *s = fs->opaque;
+    AVFilterLink *outlink = ctx->outputs[0];
+    AVFrame *in, *out, *source, *target;
+    ThreadData td;
+    int ret;
+
+    switch (s->kernel_type) {
+    case EUCLIDEAN:
+        s->kernel = euclidean_kernel;
+        break;
+    case WEUCLIDEAN:
+        s->kernel = weuclidean_kernel;
+        break;
+    default:
+        return AVERROR_BUG;
+    }
+
+    if ((ret = ff_framesync_get_frame(&s->fs, 0, &in,     1)) < 0 ||
+        (ret = ff_framesync_get_frame(&s->fs, 1, &source, 0)) < 0 ||
+        (ret = ff_framesync_get_frame(&s->fs, 2, &target, 0)) < 0)
+        return ret;
+
+    import_map(ctx->inputs[1], source);
+    import_map(ctx->inputs[2], target);
+
+    if (s->changed[0] || s->changed[1]) {
+        build_map(ctx);
+        s->changed[0] = s->changed[1] = 0;
+    }
+
+    if (!ctx->is_disabled) {
+        if (av_frame_is_writable(in)) {
+            out = in;
+        } else {
+            out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+            if (!out) {
+                av_frame_free(&in);
+                return AVERROR(ENOMEM);
+            }
+            av_frame_copy_props(out, in);
+        }
+
+        td.in = in;
+        td.out = out;
+        ff_filter_execute(ctx, colormap_slice, &td, NULL,
+                          FFMIN(in->height, ff_filter_get_nb_threads(ctx)));
+
+        if (out != in)
+            av_frame_free(&in);
+    } else {
+        out = in;
+    }
+
+    out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
+
+    return ff_filter_frame(outlink, out);
+}
+
+static int config_output(AVFilterLink *outlink)
+{
+    AVFilterContext *ctx = outlink->src;
+    ColorMapContext *s = ctx->priv;
+    AVFilterLink *inlink = ctx->inputs[0];
+    AVFilterLink *source = ctx->inputs[1];
+    AVFilterLink *target = ctx->inputs[2];
+    FFFrameSyncIn *in;
+    int ret;
+
+    outlink->time_base = inlink->time_base;
+    outlink->frame_rate = inlink->frame_rate;
+    outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
+    outlink->w = inlink->w;
+    outlink->h = inlink->h;
+
+    if ((ret = ff_framesync_init(&s->fs, ctx, 3)) < 0)
+        return ret;
+
+    in = s->fs.in;
+    in[0].time_base = inlink->time_base;
+    in[1].time_base = source->time_base;
+    in[2].time_base = target->time_base;
+    in[0].sync   = 1;
+    in[0].before = EXT_STOP;
+    in[0].after  = EXT_INFINITY;
+    in[1].sync   = 1;
+    in[1].before = EXT_STOP;
+    in[1].after  = EXT_INFINITY;
+    in[2].sync   = 1;
+    in[2].before = EXT_STOP;
+    in[2].after  = EXT_INFINITY;
+    s->fs.opaque   = s;
+    s->fs.on_event = process_frame;
+
+    ret = ff_framesync_configure(&s->fs);
+    outlink->time_base = s->fs.time_base;
+
+    return ret;
+}
+
+static int activate(AVFilterContext *ctx)
+{
+    ColorMapContext *s = ctx->priv;
+    return ff_framesync_activate(&s->fs);
+}
+
+static const AVFilterPad inputs[] = {
+    {
+        .name = "default",
+        .type = AVMEDIA_TYPE_VIDEO,
+    },
+    {
+        .name = "source",
+        .type = AVMEDIA_TYPE_VIDEO,
+    },
+    {
+        .name = "target",
+        .type = AVMEDIA_TYPE_VIDEO,
+    },
+};
+
+static const AVFilterPad outputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .config_props = config_output,
+    },
+};
+
+AVFILTER_DEFINE_CLASS(colormap);
+
+const AVFilter ff_vf_colormap = {
+    .name          = "colormap",
+    .description   = NULL_IF_CONFIG_SMALL("Apply custom Color Maps to video stream."),
+    .priv_class    = &colormap_class,
+    .priv_size     = sizeof(ColorMapContext),
+    .activate      = activate,
+    FILTER_INPUTS(inputs),
+    FILTER_OUTPUTS(outputs),
+    FILTER_PIXFMTS(AV_PIX_FMT_GBRPF32, AV_PIX_FMT_GBRAPF32),
+    .flags         = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL |
+                     AVFILTER_FLAG_SLICE_THREADS,
+    .process_command = ff_filter_process_command,
+};