videotoolbox: add alpha support

libavcodec/videotoolbox.c

@@ -1075,6 +1075,9 @@ static enum AVPixelFormat videotoolbox_best_pixel_format(AVCodecContext *avctx)
 
     int depth = descriptor->comp[0].depth;
 
+    if (descriptor->flags & AV_PIX_FMT_FLAG_ALPHA)
+        return AV_PIX_FMT_AYUV64;
+
 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
     if (depth > 10)
         return descriptor->log2_chroma_w == 0 ? AV_PIX_FMT_P416 : AV_PIX_FMT_P216;

libavutil/hwcontext_videotoolbox.c

@@ -48,6 +48,7 @@ static const struct {
 #ifdef kCFCoreFoundationVersionNumber10_7
     { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,  false, AV_PIX_FMT_NV12 },
     { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,   true,  AV_PIX_FMT_NV12 },
+    { kCVPixelFormatType_4444AYpCbCr16,                 false, AV_PIX_FMT_AYUV64 },
 #endif
 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
     { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P010 },
@@ -80,6 +81,7 @@ static const struct {
 static const enum AVPixelFormat supported_formats[] = {
 #ifdef kCFCoreFoundationVersionNumber10_7
     AV_PIX_FMT_NV12,
+    AV_PIX_FMT_AYUV64,
 #endif
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_UYVY422,