swscale/alphablend: Support chroma subsampling

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libswscale/alphablend.c

@@ -48,8 +48,54 @@ int ff_sws_alphablendaway(SwsContext *c, const uint8_t *src[],
     if (desc->flags & AV_PIX_FMT_FLAG_PLANAR) {
         for (plane = 0; plane < plane_count; plane++) {
             int w = plane ? c->chrSrcW : c->srcW;
+            int x_subsample = plane ? desc->log2_chroma_w: 0;
             int y_subsample = plane ? desc->log2_chroma_h: 0;
             for (y = srcSliceY >> y_subsample; y < FF_CEIL_RSHIFT(srcSliceH, y_subsample); y++) {
+                if (x_subsample || y_subsample) {
+                    int alpha;
+                    unsigned u;
+                    if (sixteen_bits) {
+                        ptrdiff_t alpha_step = srcStride[plane_count] >> 1;
+                        const uint16_t *s = src[plane      ] + srcStride[plane] * y;
+                        const uint16_t *a = src[plane_count] + (srcStride[plane_count] * y << y_subsample);
+                              uint16_t *d = dst[plane      ] + dstStride[plane] * y;
+                        if ((!isBE(c->srcFormat)) == !HAVE_BIGENDIAN) {
+                            for (x = 0; x < w; x++) {
+                                if (y_subsample) {
+                                    alpha = (a[2*x]              + a[2*x + 1] + 2 +
+                                             a[2*x + alpha_step] + a[2*x + alpha_step + 1]) >> 2;
+                                } else
+                                    alpha = (a[2*x] + a[2*x + 1]) >> 1;
+                                u = s[x]*alpha + target_table[((x^y)>>5)&1][plane]*(max-alpha) + off;
+                                d[x] = av_clip((u + (u >> shift)) >> shift, 0, max);
+                            }
+                        } else {
+                            for (x = 0; x < w; x++) {
+                                if (y_subsample) {
+                                    alpha = (av_bswap16(a[2*x])              + av_bswap16(a[2*x + 1]) + 2 +
+                                             av_bswap16(a[2*x + alpha_step]) + av_bswap16(a[2*x + alpha_step + 1])) >> 2;
+                                } else
+                                    alpha = (av_bswap16(a[2*x]) + av_bswap16(a[2*x + 1])) >> 1;
+                                u = av_bswap16(s[x])*alpha + target_table[((x^y)>>5)&1][plane]*(max-alpha) + off;
+                                d[x] = av_clip((u + (u >> shift)) >> shift, 0, max);
+                            }
+                        }
+                    } else {
+                        ptrdiff_t alpha_step = srcStride[plane_count];
+                        const uint8_t *s = src[plane      ] + srcStride[plane] * y;
+                        const uint8_t *a = src[plane_count] + (srcStride[plane_count] * y << y_subsample);
+                              uint8_t *d = dst[plane      ] + dstStride[plane] * y;
+                        for (x = 0; x < w; x++) {
+                            if (y_subsample) {
+                                alpha = (a[2*x]              + a[2*x + 1] + 2 +
+                                         a[2*x + alpha_step] + a[2*x + alpha_step + 1]) >> 2;
+                            } else
+                                alpha = (a[2*x] + a[2*x + 1]) >> 1;
+                            u = s[x]*alpha + target_table[((x^y)>>5)&1][plane]*(255-alpha) + 128;
+                            d[x] = (257*u) >> 16;
+                        }
+                    }
+                } else {
                 if (sixteen_bits) {
                     const uint16_t *s = src[plane      ] + srcStride[plane] * y;
                     const uint16_t *a = src[plane_count] + srcStride[plane_count] * y;
@@ -75,6 +121,7 @@ int ff_sws_alphablendaway(SwsContext *c, const uint8_t *src[],
                         d[x] = (257*u) >> 16;
                     }
                 }
+                }
             }
         }
     } else {

libswscale/utils.c

@@ -987,9 +987,9 @@ static enum AVPixelFormat alphaless_fmt(enum AVPixelFormat fmt)
     case AV_PIX_FMT_ABGR:       return AV_PIX_FMT_BGR24;
     case AV_PIX_FMT_BGRA:       return AV_PIX_FMT_BGR24;
     case AV_PIX_FMT_YA8:        return AV_PIX_FMT_GRAY8;
-//
-//     case AV_PIX_FMT_YUVA420P:   return AV_PIX_FMT_YUV420P;
-//     case AV_PIX_FMT_YUVA422P:   return AV_PIX_FMT_YUV422P;
+
+    case AV_PIX_FMT_YUVA420P:   return AV_PIX_FMT_YUV420P;
+    case AV_PIX_FMT_YUVA422P:   return AV_PIX_FMT_YUV422P;
     case AV_PIX_FMT_YUVA444P:           return AV_PIX_FMT_YUV444P;
 
     case AV_PIX_FMT_GBRAP:              return AV_PIX_FMT_GBRP;
@@ -1005,23 +1005,23 @@ static enum AVPixelFormat alphaless_fmt(enum AVPixelFormat fmt)
     case AV_PIX_FMT_YA16BE:             return AV_PIX_FMT_GRAY16;
     case AV_PIX_FMT_YA16LE:             return AV_PIX_FMT_GRAY16;
 
-//     case AV_PIX_FMT_YUVA420P9BE:        return AV_PIX_FMT_YUV420P9;
-//     case AV_PIX_FMT_YUVA422P9BE:        return AV_PIX_FMT_YUV422P9;
+    case AV_PIX_FMT_YUVA420P9BE:        return AV_PIX_FMT_YUV420P9;
+    case AV_PIX_FMT_YUVA422P9BE:        return AV_PIX_FMT_YUV422P9;
     case AV_PIX_FMT_YUVA444P9BE:        return AV_PIX_FMT_YUV444P9;
-//     case AV_PIX_FMT_YUVA420P9LE:        return AV_PIX_FMT_YUV420P9;
-//     case AV_PIX_FMT_YUVA422P9LE:        return AV_PIX_FMT_YUV422P9;
+    case AV_PIX_FMT_YUVA420P9LE:        return AV_PIX_FMT_YUV420P9;
+    case AV_PIX_FMT_YUVA422P9LE:        return AV_PIX_FMT_YUV422P9;
     case AV_PIX_FMT_YUVA444P9LE:        return AV_PIX_FMT_YUV444P9;
-//     case AV_PIX_FMT_YUVA420P10BE:       return AV_PIX_FMT_YUV420P10;
-//     case AV_PIX_FMT_YUVA422P10BE:       return AV_PIX_FMT_YUV422P10;
+    case AV_PIX_FMT_YUVA420P10BE:       return AV_PIX_FMT_YUV420P10;
+    case AV_PIX_FMT_YUVA422P10BE:       return AV_PIX_FMT_YUV422P10;
     case AV_PIX_FMT_YUVA444P10BE:       return AV_PIX_FMT_YUV444P10;
-//     case AV_PIX_FMT_YUVA420P10LE:       return AV_PIX_FMT_YUV420P10;
-//     case AV_PIX_FMT_YUVA422P10LE:       return AV_PIX_FMT_YUV422P10;
+    case AV_PIX_FMT_YUVA420P10LE:       return AV_PIX_FMT_YUV420P10;
+    case AV_PIX_FMT_YUVA422P10LE:       return AV_PIX_FMT_YUV422P10;
     case AV_PIX_FMT_YUVA444P10LE:       return AV_PIX_FMT_YUV444P10;
-//     case AV_PIX_FMT_YUVA420P16BE:       return AV_PIX_FMT_YUV420P16;
-//     case AV_PIX_FMT_YUVA422P16BE:       return AV_PIX_FMT_YUV422P16;
+    case AV_PIX_FMT_YUVA420P16BE:       return AV_PIX_FMT_YUV420P16;
+    case AV_PIX_FMT_YUVA422P16BE:       return AV_PIX_FMT_YUV422P16;
     case AV_PIX_FMT_YUVA444P16BE:       return AV_PIX_FMT_YUV444P16;
-//     case AV_PIX_FMT_YUVA420P16LE:       return AV_PIX_FMT_YUV420P16;
-//     case AV_PIX_FMT_YUVA422P16LE:       return AV_PIX_FMT_YUV422P16;
+    case AV_PIX_FMT_YUVA420P16LE:       return AV_PIX_FMT_YUV420P16;
+    case AV_PIX_FMT_YUVA422P16LE:       return AV_PIX_FMT_YUV422P16;
     case AV_PIX_FMT_YUVA444P16LE:       return AV_PIX_FMT_YUV444P16;
 
 //     case AV_PIX_FMT_AYUV64LE: