swscale: use 16-bit intermediate precision for RGB/XYZ conversion

The current logic uses 12-bit linear light math, which is woefully insufficient
and leads to nasty postarization artifacts. This patch simply switches the
internal logic to 16-bit precision.

This raises the memory requirement of these tables from 32 kB to 272 kB.

All relevant FATE tests updated for improved accuracy.

Fixes: #4829
Signed-off-by: Niklas Haas <git@haasn.dev>
Sponsored-by: Sovereign Tech Fund

libswscale/swscale.c

@@ -773,10 +773,10 @@ void ff_xyz12Torgb48(const SwsInternal *c, uint8_t *dst, int dst_stride,
                 c->xyz2rgb_matrix[2][1] * y +
                 c->xyz2rgb_matrix[2][2] * z >> 12;
 
-            // limit values to 12-bit depth
-            r = av_clip_uintp2(r, 12);
-            g = av_clip_uintp2(g, 12);
-            b = av_clip_uintp2(b, 12);
+            // limit values to 16-bit depth
+            r = av_clip_uint16(r);
+            g = av_clip_uint16(g);
+            b = av_clip_uint16(b);
 
             // convert from sRGBlinear to RGB and scale from 12bit to 16bit
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {
@@ -832,10 +832,10 @@ void ff_rgb48Toxyz12(const SwsInternal *c, uint8_t *dst, int dst_stride,
                 c->rgb2xyz_matrix[2][1] * g +
                 c->rgb2xyz_matrix[2][2] * b >> 12;
 
-            // limit values to 12-bit depth
-            x = av_clip_uintp2(x, 12);
-            y = av_clip_uintp2(y, 12);
-            z = av_clip_uintp2(z, 12);
+            // limit values to 16-bit depth
+            x = av_clip_uint16(x);
+            y = av_clip_uint16(y);
+            z = av_clip_uint16(z);
 
             // convert from XYZlinear to X'Y'Z' and scale from 12bit to 16bit
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {

libswscale/swscale_internal.h

@@ -547,10 +547,10 @@ struct SwsInternal {
 /* pre defined color-spaces gamma */
 #define XYZ_GAMMA (2.6f)
 #define RGB_GAMMA (2.2f)
-    int16_t *xyzgamma;
-    int16_t *rgbgamma;
-    int16_t *xyzgammainv;
-    int16_t *rgbgammainv;
+    uint16_t *xyzgamma;
+    uint16_t *rgbgamma;
+    uint16_t *xyzgammainv;
+    uint16_t *rgbgammainv;
     int16_t xyz2rgb_matrix[3][4];
     int16_t rgb2xyz_matrix[3][4];
 

libswscale/utils.c

@@ -951,7 +951,8 @@ static void fill_xyztables(SwsInternal *c)
         {1689, 1464,  739},
         { 871, 2929,  296},
         {  79,  488, 3891} };
-    static int16_t xyzgamma_tab[4096], rgbgamma_tab[4096], xyzgammainv_tab[4096], rgbgammainv_tab[4096];
+    static uint16_t xyzgamma_tab[4096],  rgbgammainv_tab[4096];
+    static uint16_t rgbgamma_tab[65536], xyzgammainv_tab[65536];
 
     memcpy(c->xyz2rgb_matrix, xyz2rgb_matrix, sizeof(c->xyz2rgb_matrix));
     memcpy(c->rgb2xyz_matrix, rgb2xyz_matrix, sizeof(c->rgb2xyz_matrix));
@@ -960,15 +961,19 @@ static void fill_xyztables(SwsInternal *c)
     c->xyzgammainv = xyzgammainv_tab;
     c->rgbgammainv = rgbgammainv_tab;
 
-    if (rgbgamma_tab[4095])
+    if (xyzgamma_tab[4095])
         return;
 
-    /* set gamma vectors */
+    /* set input gamma vectors */
     for (i = 0; i < 4096; i++) {
-        xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 4095.0);
-        rgbgamma_tab[i] = lrint(pow(i / 4095.0, rgbgamma) * 4095.0);
-        xyzgammainv_tab[i] = lrint(pow(i / 4095.0, xyzgammainv) * 4095.0);
-        rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 4095.0);
+        xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 65535.0);
+        rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 65535.0);
+    }
+
+    /* set output gamma vectors */
+    for (i = 0; i < 65536; i++) {
+        rgbgamma_tab[i] = lrint(pow(i / 65535.0, rgbgamma) * 4095.0);
+        xyzgammainv_tab[i] = lrint(pow(i / 65535.0, xyzgammainv) * 4095.0);
     }
 }
 

tests/ref/fate/filter-pixdesc-xyz12be

@@ -1 +1 @@
-pixdesc-xyz12be     4ec824668b9753e26c1bccffca866e27
+pixdesc-xyz12be     1508a33dea936c45d9ee13f7743af00d

tests/ref/fate/filter-pixdesc-xyz12le

@@ -1 +1 @@
-pixdesc-xyz12le     88d2563589044a3e28f6cde9a43599f9
+pixdesc-xyz12le     da2d1326fa5747a7f6ce5ac1e1494aea

tests/ref/fate/filter-pixfmts-copy

@@ -111,8 +111,8 @@ xv36be              9f556ee59a672fd8725f0bb36ce3e4b0
 xv36le              e08dcbde02f1c28a3554f372ad1278e2
 xv48be              ce34993b4b4411bba1d852b9b86aa39e
 xv48le              df913a7e61b162aa98303e5393e60c63
-xyz12be             a1ef56bf746d71f59669c28e48fc8450
-xyz12le             831ff03c1ba4ef19374686f16a064d8c
+xyz12be             f257f86373207af8aed0a1a05171df3b
+xyz12le             7922f99edc44a2c26a25becbea9914cc
 y210le              04e9487b6cce38e7531437e946cdd586
 y212le              825768be8fe92708ae80be84855066ed
 y216le              0e99aeddfee304e72d525d72998d9e9b

tests/ref/fate/filter-pixfmts-crop

@@ -108,8 +108,8 @@ xv36be              23b6f253fcb375e4145cfcb562268c5f
 xv36le              778286003497f92b84d0bd8258d6b85d
 xv48be              c90889b2cf54cc78bd58e8c47d4eb791
 xv48le              2c15c1254449ec5f9135ae61bdf4e1d5
-xyz12be             cb4571f9aaa7b59f999ef327276104b7
-xyz12le             cd6aae8d26b18bdb4b9d068586276d91
+xyz12be             e2f9f6a1ec205ab675a5a1c9521dfa6c
+xyz12le             fea1da11c07736303b139bc52b7d4759
 ya16be              071add03126a11dc6a06209e9b409f8d
 ya16le              b723211dc0647c944768c6e45e066b36
 ya8                 51a8dd297e35d40b06d3ebe8f4717895

tests/ref/fate/filter-pixfmts-field

@@ -111,8 +111,8 @@ xv36be              bcc7bda2d0a5d43db4464af6a4cb5d65
 xv36le              ba99f258370f2a56993e8760e6b30194
 xv48be              2abcd986a34789ba4310be3969020d0d
 xv48le              f6f2e33f260f48334197538f3331f7bc
-xyz12be             d2fa69ec91d3ed862f2dac3f8e7a3437
-xyz12le             02bccd5e0b6824779a1f848b0ea3e3b5
+xyz12be             3b6eb75517263b9e54b9bfa869de394f
+xyz12le             27d1d6a488cbc5d53e8d12fa0e162ddb
 y210le              4c2fba1dc40322584977d15dd07c9146
 y212le              ac2a47c45187dd54d0f55293cbffd954
 y216le              e65b5bfae1b40edbbed2012e9cd45e31

tests/ref/fate/filter-pixfmts-fieldorder

@@ -100,8 +100,8 @@ xv36be              962386c88268f4382004c3a7a82c5eb8
 xv36le              bcceffc985aaa8414c4b8072aa0889bd
 xv48be              4d6e4004b03767f12df8bb4e76c98ddf
 xv48le              9e94d82461a2131063157ac0dbe9467b
-xyz12be             15f5cda71de5fef9cec5e75e3833b6bc
-xyz12le             7be6c8781f38c21a6b8f602f62ca31e6
+xyz12be             ba6928f85c202cd77e216934f6bf0698
+xyz12le             964680cd3f3db8a7ef5510f90196961a
 y210le              22b1a02a39c4b325726bf8793bf1e8f2
 y212le              2f08fb195b948056c844acb1eee8d649
 y216le              360cb98ac80b13d3a8ec61c9f1ff3bac

tests/ref/fate/filter-pixfmts-hflip

@@ -108,8 +108,8 @@ xv36be              98f578df965eed369f46cb135e2d1345
 xv36le              e478b4b54698beb3ce1b9a2dd691d544
 xv48be              e030a2c7b1b600cfacb691b6e90c2e3d
 xv48le              fbd7f8c65cd6fc9f9108dc9a1f977dc3
-xyz12be             25f90259ff8a226befdaec3dfe82996e
-xyz12le             926c0791d59aaff61b2778e8ada3316d
+xyz12be             3c50a51a3c486a0c6853e4bbbcf3f244
+xyz12le             e020897d826ea20ded16f30ea1eb018d
 ya16be              70fa41c32ecaf3370edc38add6096db2
 ya16le              3b2c20f9e80717628ced6c6468507f63
 ya8                 4ad5920716de3d2fbbc49f95adb60345