vmnc.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529
  1. /*
  2. * VMware Screen Codec (VMnc) decoder
  3. * Copyright (c) 2006 Konstantin Shishkov
  4. *
  5. * This file is part of FFmpeg.
  6. *
  7. * FFmpeg is free software; you can redistribute it and/or
  8. * modify it under the terms of the GNU Lesser General Public
  9. * License as published by the Free Software Foundation; either
  10. * version 2.1 of the License, or (at your option) any later version.
  11. *
  12. * FFmpeg is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. * Lesser General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU Lesser General Public
  18. * License along with FFmpeg; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20. */
  21. /**
  22. * @file libavcodec/vmnc.c
  23. * VMware Screen Codec (VMnc) decoder
  24. * As Alex Beregszaszi discovered, this is effectively RFB data dump
  25. */
  26. #include <stdio.h>
  27. #include <stdlib.h>
  28. #include "libavutil/intreadwrite.h"
  29. #include "avcodec.h"
  30. enum EncTypes {
  31. MAGIC_WMVd = 0x574D5664,
  32. MAGIC_WMVe,
  33. MAGIC_WMVf,
  34. MAGIC_WMVg,
  35. MAGIC_WMVh,
  36. MAGIC_WMVi,
  37. MAGIC_WMVj
  38. };
  39. enum HexTile_Flags {
  40. HT_RAW = 1, // tile is raw
  41. HT_BKG = 2, // background color is present
  42. HT_FG = 4, // foreground color is present
  43. HT_SUB = 8, // subrects are present
  44. HT_CLR = 16 // each subrect has own color
  45. };
  46. /*
  47. * Decoder context
  48. */
  49. typedef struct VmncContext {
  50. AVCodecContext *avctx;
  51. AVFrame pic;
  52. int bpp;
  53. int bpp2;
  54. int bigendian;
  55. uint8_t pal[768];
  56. int width, height;
  57. /* cursor data */
  58. int cur_w, cur_h;
  59. int cur_x, cur_y;
  60. int cur_hx, cur_hy;
  61. uint8_t* curbits, *curmask;
  62. uint8_t* screendta;
  63. } VmncContext;
  64. /* read pixel value from stream */
  65. static av_always_inline int vmnc_get_pixel(const uint8_t* buf, int bpp, int be) {
  66. switch(bpp * 2 + be) {
  67. case 2:
  68. case 3: return *buf;
  69. case 4: return AV_RL16(buf);
  70. case 5: return AV_RB16(buf);
  71. case 8: return AV_RL32(buf);
  72. case 9: return AV_RB32(buf);
  73. default: return 0;
  74. }
  75. }
  76. static void load_cursor(VmncContext *c, const uint8_t *src)
  77. {
  78. int i, j, p;
  79. const int bpp = c->bpp2;
  80. uint8_t *dst8 = c->curbits;
  81. uint16_t *dst16 = (uint16_t*)c->curbits;
  82. uint32_t *dst32 = (uint32_t*)c->curbits;
  83. for(j = 0; j < c->cur_h; j++) {
  84. for(i = 0; i < c->cur_w; i++) {
  85. p = vmnc_get_pixel(src, bpp, c->bigendian);
  86. src += bpp;
  87. if(bpp == 1) *dst8++ = p;
  88. if(bpp == 2) *dst16++ = p;
  89. if(bpp == 4) *dst32++ = p;
  90. }
  91. }
  92. dst8 = c->curmask;
  93. dst16 = (uint16_t*)c->curmask;
  94. dst32 = (uint32_t*)c->curmask;
  95. for(j = 0; j < c->cur_h; j++) {
  96. for(i = 0; i < c->cur_w; i++) {
  97. p = vmnc_get_pixel(src, bpp, c->bigendian);
  98. src += bpp;
  99. if(bpp == 1) *dst8++ = p;
  100. if(bpp == 2) *dst16++ = p;
  101. if(bpp == 4) *dst32++ = p;
  102. }
  103. }
  104. }
  105. static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
  106. {
  107. int i, j;
  108. int w, h, x, y;
  109. w = c->cur_w;
  110. if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x;
  111. h = c->cur_h;
  112. if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y;
  113. x = c->cur_x;
  114. y = c->cur_y;
  115. if(x < 0) {
  116. w += x;
  117. x = 0;
  118. }
  119. if(y < 0) {
  120. h += y;
  121. y = 0;
  122. }
  123. if((w < 1) || (h < 1)) return;
  124. dst += x * c->bpp2 + y * stride;
  125. if(c->bpp2 == 1) {
  126. uint8_t* cd = c->curbits, *msk = c->curmask;
  127. for(j = 0; j < h; j++) {
  128. for(i = 0; i < w; i++)
  129. dst[i] = (dst[i] & cd[i]) ^ msk[i];
  130. msk += c->cur_w;
  131. cd += c->cur_w;
  132. dst += stride;
  133. }
  134. } else if(c->bpp2 == 2) {
  135. uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
  136. uint16_t* dst2;
  137. for(j = 0; j < h; j++) {
  138. dst2 = (uint16_t*)dst;
  139. for(i = 0; i < w; i++)
  140. dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
  141. msk += c->cur_w;
  142. cd += c->cur_w;
  143. dst += stride;
  144. }
  145. } else if(c->bpp2 == 4) {
  146. uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
  147. uint32_t* dst2;
  148. for(j = 0; j < h; j++) {
  149. dst2 = (uint32_t*)dst;
  150. for(i = 0; i < w; i++)
  151. dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
  152. msk += c->cur_w;
  153. cd += c->cur_w;
  154. dst += stride;
  155. }
  156. }
  157. }
  158. /* fill rectangle with given color */
  159. static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
  160. {
  161. int i, j;
  162. dst += dx * bpp + dy * stride;
  163. if(bpp == 1){
  164. for(j = 0; j < h; j++) {
  165. memset(dst, color, w);
  166. dst += stride;
  167. }
  168. }else if(bpp == 2){
  169. uint16_t* dst2;
  170. for(j = 0; j < h; j++) {
  171. dst2 = (uint16_t*)dst;
  172. for(i = 0; i < w; i++) {
  173. *dst2++ = color;
  174. }
  175. dst += stride;
  176. }
  177. }else if(bpp == 4){
  178. uint32_t* dst2;
  179. for(j = 0; j < h; j++) {
  180. dst2 = (uint32_t*)dst;
  181. for(i = 0; i < w; i++) {
  182. dst2[i] = color;
  183. }
  184. dst += stride;
  185. }
  186. }
  187. }
  188. static av_always_inline void paint_raw(uint8_t *dst, int w, int h, const uint8_t* src, int bpp, int be, int stride)
  189. {
  190. int i, j, p;
  191. for(j = 0; j < h; j++) {
  192. for(i = 0; i < w; i++) {
  193. p = vmnc_get_pixel(src, bpp, be);
  194. src += bpp;
  195. switch(bpp){
  196. case 1:
  197. dst[i] = p;
  198. break;
  199. case 2:
  200. ((uint16_t*)dst)[i] = p;
  201. break;
  202. case 4:
  203. ((uint32_t*)dst)[i] = p;
  204. break;
  205. }
  206. }
  207. dst += stride;
  208. }
  209. }
  210. static int decode_hextile(VmncContext *c, uint8_t* dst, const uint8_t* src, int ssize, int w, int h, int stride)
  211. {
  212. int i, j, k;
  213. int bg = 0, fg = 0, rects, color, flags, xy, wh;
  214. const int bpp = c->bpp2;
  215. uint8_t *dst2;
  216. int bw = 16, bh = 16;
  217. const uint8_t *ssrc=src;
  218. for(j = 0; j < h; j += 16) {
  219. dst2 = dst;
  220. bw = 16;
  221. if(j + 16 > h) bh = h - j;
  222. for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
  223. if(src - ssrc >= ssize) {
  224. av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
  225. return -1;
  226. }
  227. if(i + 16 > w) bw = w - i;
  228. flags = *src++;
  229. if(flags & HT_RAW) {
  230. if(src - ssrc > ssize - bw * bh * bpp) {
  231. av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
  232. return -1;
  233. }
  234. paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
  235. src += bw * bh * bpp;
  236. } else {
  237. if(flags & HT_BKG) {
  238. bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
  239. }
  240. if(flags & HT_FG) {
  241. fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
  242. }
  243. rects = 0;
  244. if(flags & HT_SUB)
  245. rects = *src++;
  246. color = !!(flags & HT_CLR);
  247. paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
  248. if(src - ssrc > ssize - rects * (color * bpp + 2)) {
  249. av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
  250. return -1;
  251. }
  252. for(k = 0; k < rects; k++) {
  253. if(color) {
  254. fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
  255. }
  256. xy = *src++;
  257. wh = *src++;
  258. if ( (xy >> 4) + (wh >> 4) + 1 > w - i
  259. || (xy & 0xF) + (wh & 0xF)+1 > h - j) {
  260. av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");
  261. return AVERROR_INVALIDDATA;
  262. }
  263. paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
  264. }
  265. }
  266. }
  267. dst += stride * 16;
  268. }
  269. return src - ssrc;
  270. }
  271. static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, const uint8_t *buf, int buf_size)
  272. {
  273. VmncContext * const c = avctx->priv_data;
  274. uint8_t *outptr;
  275. const uint8_t *src = buf;
  276. int dx, dy, w, h, depth, enc, chunks, res, size_left;
  277. c->pic.reference = 1;
  278. c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
  279. if(avctx->reget_buffer(avctx, &c->pic) < 0){
  280. av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
  281. return -1;
  282. }
  283. c->pic.key_frame = 0;
  284. c->pic.pict_type = FF_P_TYPE;
  285. //restore screen after cursor
  286. if(c->screendta) {
  287. int i;
  288. w = c->cur_w;
  289. if(c->width < c->cur_x + w) w = c->width - c->cur_x;
  290. h = c->cur_h;
  291. if(c->height < c->cur_y + h) h = c->height - c->cur_y;
  292. dx = c->cur_x;
  293. if(dx < 0) {
  294. w += dx;
  295. dx = 0;
  296. }
  297. dy = c->cur_y;
  298. if(dy < 0) {
  299. h += dy;
  300. dy = 0;
  301. }
  302. if((w > 0) && (h > 0)) {
  303. outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
  304. for(i = 0; i < h; i++) {
  305. memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2);
  306. outptr += c->pic.linesize[0];
  307. }
  308. }
  309. }
  310. src += 2;
  311. chunks = AV_RB16(src); src += 2;
  312. while(chunks--) {
  313. dx = AV_RB16(src); src += 2;
  314. dy = AV_RB16(src); src += 2;
  315. w = AV_RB16(src); src += 2;
  316. h = AV_RB16(src); src += 2;
  317. enc = AV_RB32(src); src += 4;
  318. outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
  319. size_left = buf_size - (src - buf);
  320. switch(enc) {
  321. case MAGIC_WMVd: // cursor
  322. if(size_left < 2 + w * h * c->bpp2 * 2) {
  323. av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
  324. return -1;
  325. }
  326. src += 2;
  327. c->cur_w = w;
  328. c->cur_h = h;
  329. c->cur_hx = dx;
  330. c->cur_hy = dy;
  331. if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
  332. av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
  333. c->cur_hx = c->cur_hy = 0;
  334. }
  335. c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2);
  336. c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2);
  337. c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2);
  338. load_cursor(c, src);
  339. src += w * h * c->bpp2 * 2;
  340. break;
  341. case MAGIC_WMVe: // unknown
  342. src += 2;
  343. break;
  344. case MAGIC_WMVf: // update cursor position
  345. c->cur_x = dx - c->cur_hx;
  346. c->cur_y = dy - c->cur_hy;
  347. break;
  348. case MAGIC_WMVg: // unknown
  349. src += 10;
  350. break;
  351. case MAGIC_WMVh: // unknown
  352. src += 4;
  353. break;
  354. case MAGIC_WMVi: // ServerInitialization struct
  355. c->pic.key_frame = 1;
  356. c->pic.pict_type = FF_I_TYPE;
  357. depth = *src++;
  358. if(depth != c->bpp) {
  359. av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
  360. }
  361. src++;
  362. c->bigendian = *src++;
  363. if(c->bigendian & (~1)) {
  364. av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
  365. return -1;
  366. }
  367. //skip the rest of pixel format data
  368. src += 13;
  369. break;
  370. case MAGIC_WMVj: // unknown
  371. src += 2;
  372. break;
  373. case 0x00000000: // raw rectangle data
  374. if((dx + w > c->width) || (dy + h > c->height)) {
  375. av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
  376. return -1;
  377. }
  378. if(size_left < w * h * c->bpp2) {
  379. av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left);
  380. return -1;
  381. }
  382. paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]);
  383. src += w * h * c->bpp2;
  384. break;
  385. case 0x00000005: // HexTile encoded rectangle
  386. if((dx + w > c->width) || (dy + h > c->height)) {
  387. av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
  388. return -1;
  389. }
  390. res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]);
  391. if(res < 0)
  392. return -1;
  393. src += res;
  394. break;
  395. default:
  396. av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
  397. chunks = 0; // leave chunks decoding loop
  398. }
  399. }
  400. if(c->screendta){
  401. int i;
  402. //save screen data before painting cursor
  403. w = c->cur_w;
  404. if(c->width < c->cur_x + w) w = c->width - c->cur_x;
  405. h = c->cur_h;
  406. if(c->height < c->cur_y + h) h = c->height - c->cur_y;
  407. dx = c->cur_x;
  408. if(dx < 0) {
  409. w += dx;
  410. dx = 0;
  411. }
  412. dy = c->cur_y;
  413. if(dy < 0) {
  414. h += dy;
  415. dy = 0;
  416. }
  417. if((w > 0) && (h > 0)) {
  418. outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
  419. for(i = 0; i < h; i++) {
  420. memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2);
  421. outptr += c->pic.linesize[0];
  422. }
  423. outptr = c->pic.data[0];
  424. put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
  425. }
  426. }
  427. *data_size = sizeof(AVFrame);
  428. *(AVFrame*)data = c->pic;
  429. /* always report that the buffer was completely consumed */
  430. return buf_size;
  431. }
  432. /*
  433. *
  434. * Init VMnc decoder
  435. *
  436. */
  437. static av_cold int decode_init(AVCodecContext *avctx)
  438. {
  439. VmncContext * const c = avctx->priv_data;
  440. c->avctx = avctx;
  441. c->pic.data[0] = NULL;
  442. c->width = avctx->width;
  443. c->height = avctx->height;
  444. if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) {
  445. return 1;
  446. }
  447. c->bpp = avctx->bits_per_coded_sample;
  448. c->bpp2 = c->bpp/8;
  449. switch(c->bpp){
  450. case 8:
  451. avctx->pix_fmt = PIX_FMT_PAL8;
  452. break;
  453. case 16:
  454. avctx->pix_fmt = PIX_FMT_RGB555;
  455. break;
  456. case 32:
  457. avctx->pix_fmt = PIX_FMT_RGB32;
  458. break;
  459. default:
  460. av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
  461. }
  462. return 0;
  463. }
  464. /*
  465. *
  466. * Uninit VMnc decoder
  467. *
  468. */
  469. static av_cold int decode_end(AVCodecContext *avctx)
  470. {
  471. VmncContext * const c = avctx->priv_data;
  472. if (c->pic.data[0])
  473. avctx->release_buffer(avctx, &c->pic);
  474. av_free(c->curbits);
  475. av_free(c->curmask);
  476. av_free(c->screendta);
  477. return 0;
  478. }
  479. AVCodec vmnc_decoder = {
  480. "vmnc",
  481. CODEC_TYPE_VIDEO,
  482. CODEC_ID_VMNC,
  483. sizeof(VmncContext),
  484. decode_init,
  485. NULL,
  486. decode_end,
  487. decode_frame,
  488. .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),
  489. };