qsvenc_hevc.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. /*
  2. * Intel MediaSDK QSV based HEVC encoder
  3. *
  4. * This file is part of FFmpeg.
  5. *
  6. * FFmpeg is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 2.1 of the License, or (at your option) any later version.
  10. *
  11. * FFmpeg is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public
  17. * License along with FFmpeg; if not, write to the Free Software
  18. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19. */
  20. #include <stdint.h>
  21. #include <sys/types.h>
  22. #include <mfxvideo.h>
  23. #include "libavutil/common.h"
  24. #include "libavutil/opt.h"
  25. #include "libavutil/mastering_display_metadata.h"
  26. #include "avcodec.h"
  27. #include "bytestream.h"
  28. #include "codec_internal.h"
  29. #include "get_bits.h"
  30. #include "hevc.h"
  31. #include "hevcdec.h"
  32. #include "h2645_parse.h"
  33. #include "qsv.h"
  34. #include "qsvenc.h"
  35. enum LoadPlugin {
  36. LOAD_PLUGIN_NONE,
  37. LOAD_PLUGIN_HEVC_SW,
  38. LOAD_PLUGIN_HEVC_HW,
  39. };
  40. typedef struct QSVHEVCEncContext {
  41. AVClass *class;
  42. QSVEncContext qsv;
  43. int load_plugin;
  44. } QSVHEVCEncContext;
  45. static int generate_fake_vps(QSVEncContext *q, AVCodecContext *avctx)
  46. {
  47. GetByteContext gbc;
  48. PutByteContext pbc;
  49. GetBitContext gb;
  50. H2645RBSP sps_rbsp = { NULL };
  51. H2645NAL sps_nal = { NULL };
  52. HEVCSPS sps = { 0 };
  53. HEVCVPS vps = { 0 };
  54. uint8_t vps_buf[128], vps_rbsp_buf[128];
  55. uint8_t *new_extradata;
  56. unsigned int sps_id;
  57. int ret, i, type, vps_size;
  58. if (!avctx->extradata_size) {
  59. av_log(avctx, AV_LOG_ERROR, "No extradata returned from libmfx\n");
  60. return AVERROR_UNKNOWN;
  61. }
  62. av_fast_padded_malloc(&sps_rbsp.rbsp_buffer, &sps_rbsp.rbsp_buffer_alloc_size, avctx->extradata_size);
  63. if (!sps_rbsp.rbsp_buffer)
  64. return AVERROR(ENOMEM);
  65. /* parse the SPS */
  66. ret = ff_h2645_extract_rbsp(avctx->extradata + 4, avctx->extradata_size - 4, &sps_rbsp, &sps_nal, 1);
  67. if (ret < 0) {
  68. av_log(avctx, AV_LOG_ERROR, "Error unescaping the SPS buffer\n");
  69. return ret;
  70. }
  71. ret = init_get_bits8(&gb, sps_nal.data, sps_nal.size);
  72. if (ret < 0) {
  73. av_freep(&sps_rbsp.rbsp_buffer);
  74. return ret;
  75. }
  76. get_bits(&gb, 1);
  77. type = get_bits(&gb, 6);
  78. if (type != HEVC_NAL_SPS) {
  79. av_log(avctx, AV_LOG_ERROR, "Unexpected NAL type in the extradata: %d\n",
  80. type);
  81. av_freep(&sps_rbsp.rbsp_buffer);
  82. return AVERROR_INVALIDDATA;
  83. }
  84. get_bits(&gb, 9);
  85. ret = ff_hevc_parse_sps(&sps, &gb, &sps_id, 0, NULL, avctx);
  86. av_freep(&sps_rbsp.rbsp_buffer);
  87. if (ret < 0) {
  88. av_log(avctx, AV_LOG_ERROR, "Error parsing the SPS\n");
  89. return ret;
  90. }
  91. /* generate the VPS */
  92. vps.vps_max_layers = 1;
  93. vps.vps_max_sub_layers = sps.max_sub_layers;
  94. vps.vps_temporal_id_nesting_flag = sps.temporal_id_nesting_flag;
  95. memcpy(&vps.ptl, &sps.ptl, sizeof(vps.ptl));
  96. vps.vps_sub_layer_ordering_info_present_flag = 1;
  97. for (i = 0; i < HEVC_MAX_SUB_LAYERS; i++) {
  98. vps.vps_max_dec_pic_buffering[i] = sps.temporal_layer[i].max_dec_pic_buffering;
  99. vps.vps_num_reorder_pics[i] = sps.temporal_layer[i].num_reorder_pics;
  100. vps.vps_max_latency_increase[i] = sps.temporal_layer[i].max_latency_increase;
  101. }
  102. vps.vps_num_layer_sets = 1;
  103. vps.vps_timing_info_present_flag = sps.vui.vui_timing_info_present_flag;
  104. vps.vps_num_units_in_tick = sps.vui.vui_num_units_in_tick;
  105. vps.vps_time_scale = sps.vui.vui_time_scale;
  106. vps.vps_poc_proportional_to_timing_flag = sps.vui.vui_poc_proportional_to_timing_flag;
  107. vps.vps_num_ticks_poc_diff_one = sps.vui.vui_num_ticks_poc_diff_one_minus1 + 1;
  108. vps.vps_num_hrd_parameters = 0;
  109. /* generate the encoded RBSP form of the VPS */
  110. ret = ff_hevc_encode_nal_vps(&vps, sps.vps_id, vps_rbsp_buf, sizeof(vps_rbsp_buf));
  111. if (ret < 0) {
  112. av_log(avctx, AV_LOG_ERROR, "Error writing the VPS\n");
  113. return ret;
  114. }
  115. /* escape and add the startcode */
  116. bytestream2_init(&gbc, vps_rbsp_buf, ret);
  117. bytestream2_init_writer(&pbc, vps_buf, sizeof(vps_buf));
  118. bytestream2_put_be32(&pbc, 1); // startcode
  119. bytestream2_put_byte(&pbc, HEVC_NAL_VPS << 1); // NAL
  120. bytestream2_put_byte(&pbc, 1); // header
  121. while (bytestream2_get_bytes_left(&gbc)) {
  122. if (bytestream2_get_bytes_left(&gbc) >= 3 && bytestream2_peek_be24(&gbc) <= 3) {
  123. bytestream2_put_be24(&pbc, 3);
  124. bytestream2_skip(&gbc, 2);
  125. } else
  126. bytestream2_put_byte(&pbc, bytestream2_get_byte(&gbc));
  127. }
  128. vps_size = bytestream2_tell_p(&pbc);
  129. new_extradata = av_mallocz(vps_size + avctx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE);
  130. if (!new_extradata)
  131. return AVERROR(ENOMEM);
  132. memcpy(new_extradata, vps_buf, vps_size);
  133. memcpy(new_extradata + vps_size, avctx->extradata, avctx->extradata_size);
  134. av_freep(&avctx->extradata);
  135. avctx->extradata = new_extradata;
  136. avctx->extradata_size += vps_size;
  137. return 0;
  138. }
  139. static int qsv_hevc_set_encode_ctrl(AVCodecContext *avctx,
  140. const AVFrame *frame, mfxEncodeCtrl *enc_ctrl)
  141. {
  142. QSVHEVCEncContext *q = avctx->priv_data;
  143. AVFrameSideData *sd;
  144. if (!frame || !QSV_RUNTIME_VERSION_ATLEAST(q->qsv.ver, 1, 25))
  145. return 0;
  146. sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
  147. if (sd) {
  148. AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *)sd->data;
  149. // SEI is needed when both the primaries and luminance are set
  150. if (mdm->has_primaries && mdm->has_luminance) {
  151. const int mapping[3] = {1, 2, 0};
  152. const int chroma_den = 50000;
  153. const int luma_den = 10000;
  154. int i;
  155. mfxExtMasteringDisplayColourVolume *mdcv = av_mallocz(sizeof(mfxExtMasteringDisplayColourVolume));
  156. if (!mdcv)
  157. return AVERROR(ENOMEM);
  158. mdcv->Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
  159. mdcv->Header.BufferSz = sizeof(*mdcv);
  160. for (i = 0; i < 3; i++) {
  161. const int j = mapping[i];
  162. mdcv->DisplayPrimariesX[i] =
  163. FFMIN(lrint(chroma_den *
  164. av_q2d(mdm->display_primaries[j][0])),
  165. chroma_den);
  166. mdcv->DisplayPrimariesY[i] =
  167. FFMIN(lrint(chroma_den *
  168. av_q2d(mdm->display_primaries[j][1])),
  169. chroma_den);
  170. }
  171. mdcv->WhitePointX =
  172. FFMIN(lrint(chroma_den * av_q2d(mdm->white_point[0])),
  173. chroma_den);
  174. mdcv->WhitePointY =
  175. FFMIN(lrint(chroma_den * av_q2d(mdm->white_point[1])),
  176. chroma_den);
  177. mdcv->MaxDisplayMasteringLuminance =
  178. lrint(luma_den * av_q2d(mdm->max_luminance));
  179. mdcv->MinDisplayMasteringLuminance =
  180. FFMIN(lrint(luma_den * av_q2d(mdm->min_luminance)),
  181. mdcv->MaxDisplayMasteringLuminance);
  182. enc_ctrl->ExtParam[enc_ctrl->NumExtParam++] = (mfxExtBuffer *)mdcv;
  183. }
  184. }
  185. sd = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
  186. if (sd) {
  187. AVContentLightMetadata *clm = (AVContentLightMetadata *)sd->data;
  188. mfxExtContentLightLevelInfo * clli = av_mallocz(sizeof(mfxExtContentLightLevelInfo));
  189. if (!clli)
  190. return AVERROR(ENOMEM);
  191. clli->Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
  192. clli->Header.BufferSz = sizeof(*clli);
  193. clli->MaxContentLightLevel = FFMIN(clm->MaxCLL, 65535);
  194. clli->MaxPicAverageLightLevel = FFMIN(clm->MaxFALL, 65535);
  195. enc_ctrl->ExtParam[enc_ctrl->NumExtParam++] = (mfxExtBuffer *)clli;
  196. }
  197. return 0;
  198. }
  199. static av_cold int qsv_enc_init(AVCodecContext *avctx)
  200. {
  201. QSVHEVCEncContext *q = avctx->priv_data;
  202. int ret;
  203. if (q->load_plugin != LOAD_PLUGIN_NONE) {
  204. static const char * const uid_hevcenc_sw = "2fca99749fdb49aeb121a5b63ef568f7";
  205. static const char * const uid_hevcenc_hw = "6fadc791a0c2eb479ab6dcd5ea9da347";
  206. if (q->qsv.load_plugins[0]) {
  207. av_log(avctx, AV_LOG_WARNING,
  208. "load_plugins is not empty, but load_plugin is not set to 'none'."
  209. "The load_plugin value will be ignored.\n");
  210. } else {
  211. av_freep(&q->qsv.load_plugins);
  212. if (q->load_plugin == LOAD_PLUGIN_HEVC_SW)
  213. q->qsv.load_plugins = av_strdup(uid_hevcenc_sw);
  214. else
  215. q->qsv.load_plugins = av_strdup(uid_hevcenc_hw);
  216. if (!q->qsv.load_plugins)
  217. return AVERROR(ENOMEM);
  218. }
  219. }
  220. // HEVC and H264 meaning of the value is shifted by 1, make it consistent
  221. q->qsv.idr_interval++;
  222. q->qsv.set_encode_ctrl_cb = qsv_hevc_set_encode_ctrl;
  223. ret = ff_qsv_enc_init(avctx, &q->qsv);
  224. if (ret < 0)
  225. return ret;
  226. if (!q->qsv.hevc_vps) {
  227. ret = generate_fake_vps(&q->qsv, avctx);
  228. if (ret < 0) {
  229. ff_qsv_enc_close(avctx, &q->qsv);
  230. return ret;
  231. }
  232. }
  233. return 0;
  234. }
  235. static int qsv_enc_frame(AVCodecContext *avctx, AVPacket *pkt,
  236. const AVFrame *frame, int *got_packet)
  237. {
  238. QSVHEVCEncContext *q = avctx->priv_data;
  239. return ff_qsv_encode(avctx, &q->qsv, pkt, frame, got_packet);
  240. }
  241. static av_cold int qsv_enc_close(AVCodecContext *avctx)
  242. {
  243. QSVHEVCEncContext *q = avctx->priv_data;
  244. return ff_qsv_enc_close(avctx, &q->qsv);
  245. }
  246. #define OFFSET(x) offsetof(QSVHEVCEncContext, x)
  247. #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
  248. static const AVOption options[] = {
  249. QSV_COMMON_OPTS
  250. QSV_OPTION_RDO
  251. QSV_OPTION_MAX_FRAME_SIZE
  252. QSV_OPTION_MAX_SLICE_SIZE
  253. QSV_OPTION_MBBRC
  254. QSV_OPTION_EXTBRC
  255. QSV_OPTION_P_STRATEGY
  256. QSV_OPTION_B_STRATEGY
  257. QSV_OPTION_DBLK_IDC
  258. QSV_OPTION_LOW_DELAY_BRC
  259. QSV_OPTION_MAX_MIN_QP
  260. QSV_OPTION_ADAPTIVE_I
  261. QSV_OPTION_ADAPTIVE_B
  262. QSV_OPTION_SCENARIO
  263. QSV_OPTION_AVBR
  264. QSV_OPTION_SKIP_FRAME
  265. { "idr_interval", "Distance (in I-frames) between IDR frames", OFFSET(qsv.idr_interval), AV_OPT_TYPE_INT, { .i64 = 0 }, -1, INT_MAX, VE, "idr_interval" },
  266. { "begin_only", "Output an IDR-frame only at the beginning of the stream", 0, AV_OPT_TYPE_CONST, { .i64 = -1 }, 0, 0, VE, "idr_interval" },
  267. { "load_plugin", "A user plugin to load in an internal session", OFFSET(load_plugin), AV_OPT_TYPE_INT, { .i64 = LOAD_PLUGIN_HEVC_HW }, LOAD_PLUGIN_NONE, LOAD_PLUGIN_HEVC_HW, VE, "load_plugin" },
  268. { "none", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = LOAD_PLUGIN_NONE }, 0, 0, VE, "load_plugin" },
  269. { "hevc_sw", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = LOAD_PLUGIN_HEVC_SW }, 0, 0, VE, "load_plugin" },
  270. { "hevc_hw", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = LOAD_PLUGIN_HEVC_HW }, 0, 0, VE, "load_plugin" },
  271. { "load_plugins", "A :-separate list of hexadecimal plugin UIDs to load in an internal session",
  272. OFFSET(qsv.load_plugins), AV_OPT_TYPE_STRING, { .str = "" }, 0, 0, VE },
  273. { "look_ahead_depth", "Depth of look ahead in number frames, available when extbrc option is enabled", OFFSET(qsv.look_ahead_depth), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 100, VE },
  274. { "profile", NULL, OFFSET(qsv.profile), AV_OPT_TYPE_INT, { .i64 = MFX_PROFILE_UNKNOWN }, 0, INT_MAX, VE, "profile" },
  275. { "unknown", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_UNKNOWN }, INT_MIN, INT_MAX, VE, "profile" },
  276. { "main", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_HEVC_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
  277. { "main10", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_HEVC_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
  278. { "mainsp", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_HEVC_MAINSP }, INT_MIN, INT_MAX, VE, "profile" },
  279. { "rext", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_HEVC_REXT }, INT_MIN, INT_MAX, VE, "profile" },
  280. #if QSV_VERSION_ATLEAST(1, 32)
  281. { "scc", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_HEVC_SCC }, INT_MIN, INT_MAX, VE, "profile" },
  282. #endif
  283. { "tier", "Set the encoding tier (only level >= 4 can support high tier)", OFFSET(qsv.tier), AV_OPT_TYPE_INT, { .i64 = MFX_TIER_HEVC_HIGH }, MFX_TIER_HEVC_MAIN, MFX_TIER_HEVC_HIGH, VE, "tier" },
  284. { "main", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_TIER_HEVC_MAIN }, INT_MIN, INT_MAX, VE, "tier" },
  285. { "high", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_TIER_HEVC_HIGH }, INT_MIN, INT_MAX, VE, "tier" },
  286. { "gpb", "1: GPB (generalized P/B frame); 0: regular P frame", OFFSET(qsv.gpb), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, VE},
  287. { "tile_cols", "Number of columns for tiled encoding", OFFSET(qsv.tile_cols), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, UINT16_MAX, VE },
  288. { "tile_rows", "Number of rows for tiled encoding", OFFSET(qsv.tile_rows), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, UINT16_MAX, VE },
  289. { "recovery_point_sei", "Insert recovery point SEI messages", OFFSET(qsv.recovery_point_sei), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 1, VE },
  290. { "aud", "Insert the Access Unit Delimiter NAL", OFFSET(qsv.aud), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE},
  291. { "pic_timing_sei", "Insert picture timing SEI with pic_struct_syntax element", OFFSET(qsv.pic_timing_sei), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, VE },
  292. { "transform_skip", "Turn this option ON to enable transformskip", OFFSET(qsv.transform_skip), AV_OPT_TYPE_INT, { .i64 = -1}, -1, 1, VE},
  293. { "int_ref_type", "Intra refresh type. B frames should be set to 0", OFFSET(qsv.int_ref_type), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, UINT16_MAX, VE, "int_ref_type" },
  294. { "none", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, .flags = VE, "int_ref_type" },
  295. { "vertical", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, .flags = VE, "int_ref_type" },
  296. { "horizontal", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, .flags = VE, "int_ref_type" },
  297. { "slice" , NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 3 }, .flags = VE, "int_ref_type" },
  298. { "int_ref_cycle_size", "Number of frames in the intra refresh cycle", OFFSET(qsv.int_ref_cycle_size), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, UINT16_MAX, VE },
  299. { "int_ref_qp_delta", "QP difference for the refresh MBs", OFFSET(qsv.int_ref_qp_delta), AV_OPT_TYPE_INT, { .i64 = INT16_MIN }, INT16_MIN, INT16_MAX, VE },
  300. { "int_ref_cycle_dist", "Distance between the beginnings of the intra-refresh cycles in frames", OFFSET(qsv.int_ref_cycle_dist), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT16_MAX, VE },
  301. { NULL },
  302. };
  303. static const AVClass class = {
  304. .class_name = "hevc_qsv encoder",
  305. .item_name = av_default_item_name,
  306. .option = options,
  307. .version = LIBAVUTIL_VERSION_INT,
  308. };
  309. static const FFCodecDefault qsv_enc_defaults[] = {
  310. { "b", "1M" },
  311. { "refs", "0" },
  312. { "g", "-1" },
  313. { "bf", "-1" },
  314. { "qmin", "-1" },
  315. { "qmax", "-1" },
  316. { "trellis", "-1" },
  317. { NULL },
  318. };
  319. const FFCodec ff_hevc_qsv_encoder = {
  320. .p.name = "hevc_qsv",
  321. CODEC_LONG_NAME("HEVC (Intel Quick Sync Video acceleration)"),
  322. .priv_data_size = sizeof(QSVHEVCEncContext),
  323. .p.type = AVMEDIA_TYPE_VIDEO,
  324. .p.id = AV_CODEC_ID_HEVC,
  325. .init = qsv_enc_init,
  326. FF_CODEC_ENCODE_CB(qsv_enc_frame),
  327. .close = qsv_enc_close,
  328. .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HYBRID,
  329. .p.pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_NV12,
  330. AV_PIX_FMT_P010,
  331. AV_PIX_FMT_P012,
  332. AV_PIX_FMT_YUYV422,
  333. AV_PIX_FMT_Y210,
  334. AV_PIX_FMT_QSV,
  335. AV_PIX_FMT_BGRA,
  336. AV_PIX_FMT_X2RGB10,
  337. AV_PIX_FMT_VUYX,
  338. AV_PIX_FMT_XV30,
  339. AV_PIX_FMT_NONE },
  340. .p.priv_class = &class,
  341. .defaults = qsv_enc_defaults,
  342. .caps_internal = FF_CODEC_CAP_NOT_INIT_THREADSAFE |
  343. FF_CODEC_CAP_INIT_CLEANUP,
  344. .p.wrapper_name = "qsv",
  345. .hw_configs = ff_qsv_enc_hw_configs,
  346. };