2 * VMware Screen Codec (VMnc) decoder
3 * Copyright (c) 2006 Konstantin Shishkov
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * @file libavcodec/vmnc.c
24 * VMware Screen Codec (VMnc) decoder
25 * As Alex Beregszaszi discovered, this is effectively RFB data dump
31 #include "libavutil/intreadwrite.h"
35 MAGIC_WMVd
= 0x574D5664,
45 HT_RAW
= 1, // tile is raw
46 HT_BKG
= 2, // background color is present
47 HT_FG
= 4, // foreground color is present
48 HT_SUB
= 8, // subrects are present
49 HT_CLR
= 16 // each subrect has own color
55 typedef struct VmncContext
{
56 AVCodecContext
*avctx
;
69 uint8_t* curbits
, *curmask
;
73 /* read pixel value from stream */
74 static av_always_inline
int vmnc_get_pixel(const uint8_t* buf
, int bpp
, int be
) {
75 switch(bpp
* 2 + be
) {
78 case 4: return AV_RL16(buf
);
79 case 5: return AV_RB16(buf
);
80 case 8: return AV_RL32(buf
);
81 case 9: return AV_RB32(buf
);
86 static void load_cursor(VmncContext
*c
, const uint8_t *src
)
89 const int bpp
= c
->bpp2
;
90 uint8_t *dst8
= c
->curbits
;
91 uint16_t *dst16
= (uint16_t*)c
->curbits
;
92 uint32_t *dst32
= (uint32_t*)c
->curbits
;
94 for(j
= 0; j
< c
->cur_h
; j
++) {
95 for(i
= 0; i
< c
->cur_w
; i
++) {
96 p
= vmnc_get_pixel(src
, bpp
, c
->bigendian
);
98 if(bpp
== 1) *dst8
++ = p
;
99 if(bpp
== 2) *dst16
++ = p
;
100 if(bpp
== 4) *dst32
++ = p
;
104 dst16
= (uint16_t*)c
->curmask
;
105 dst32
= (uint32_t*)c
->curmask
;
106 for(j
= 0; j
< c
->cur_h
; j
++) {
107 for(i
= 0; i
< c
->cur_w
; i
++) {
108 p
= vmnc_get_pixel(src
, bpp
, c
->bigendian
);
110 if(bpp
== 1) *dst8
++ = p
;
111 if(bpp
== 2) *dst16
++ = p
;
112 if(bpp
== 4) *dst32
++ = p
;
117 static void put_cursor(uint8_t *dst
, int stride
, VmncContext
*c
, int dx
, int dy
)
122 if(c
->width
< c
->cur_x
+ c
->cur_w
) w
= c
->width
- c
->cur_x
;
124 if(c
->height
< c
->cur_y
+ c
->cur_h
) h
= c
->height
- c
->cur_y
;
136 if((w
< 1) || (h
< 1)) return;
137 dst
+= x
* c
->bpp2
+ y
* stride
;
140 uint8_t* cd
= c
->curbits
, *msk
= c
->curmask
;
141 for(j
= 0; j
< h
; j
++) {
142 for(i
= 0; i
< w
; i
++)
143 dst
[i
] = (dst
[i
] & cd
[i
]) ^ msk
[i
];
148 } else if(c
->bpp2
== 2) {
149 uint16_t* cd
= (uint16_t*)c
->curbits
, *msk
= (uint16_t*)c
->curmask
;
151 for(j
= 0; j
< h
; j
++) {
152 dst2
= (uint16_t*)dst
;
153 for(i
= 0; i
< w
; i
++)
154 dst2
[i
] = (dst2
[i
] & cd
[i
]) ^ msk
[i
];
159 } else if(c
->bpp2
== 4) {
160 uint32_t* cd
= (uint32_t*)c
->curbits
, *msk
= (uint32_t*)c
->curmask
;
162 for(j
= 0; j
< h
; j
++) {
163 dst2
= (uint32_t*)dst
;
164 for(i
= 0; i
< w
; i
++)
165 dst2
[i
] = (dst2
[i
] & cd
[i
]) ^ msk
[i
];
173 /* fill rectangle with given color */
174 static av_always_inline
void paint_rect(uint8_t *dst
, int dx
, int dy
, int w
, int h
, int color
, int bpp
, int stride
)
177 dst
+= dx
* bpp
+ dy
* stride
;
179 for(j
= 0; j
< h
; j
++) {
180 memset(dst
, color
, w
);
185 for(j
= 0; j
< h
; j
++) {
186 dst2
= (uint16_t*)dst
;
187 for(i
= 0; i
< w
; i
++) {
194 for(j
= 0; j
< h
; j
++) {
195 dst2
= (uint32_t*)dst
;
196 for(i
= 0; i
< w
; i
++) {
204 static av_always_inline
void paint_raw(uint8_t *dst
, int w
, int h
, const uint8_t* src
, int bpp
, int be
, int stride
)
207 for(j
= 0; j
< h
; j
++) {
208 for(i
= 0; i
< w
; i
++) {
209 p
= vmnc_get_pixel(src
, bpp
, be
);
216 ((uint16_t*)dst
)[i
] = p
;
219 ((uint32_t*)dst
)[i
] = p
;
227 static int decode_hextile(VmncContext
*c
, uint8_t* dst
, const uint8_t* src
, int ssize
, int w
, int h
, int stride
)
230 int bg
= 0, fg
= 0, rects
, color
, flags
, xy
, wh
;
231 const int bpp
= c
->bpp2
;
233 int bw
= 16, bh
= 16;
234 const uint8_t *ssrc
=src
;
236 for(j
= 0; j
< h
; j
+= 16) {
239 if(j
+ 16 > h
) bh
= h
- j
;
240 for(i
= 0; i
< w
; i
+= 16, dst2
+= 16 * bpp
) {
241 if(src
- ssrc
>= ssize
) {
242 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
245 if(i
+ 16 > w
) bw
= w
- i
;
248 if(src
- ssrc
> ssize
- bw
* bh
* bpp
) {
249 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
252 paint_raw(dst2
, bw
, bh
, src
, bpp
, c
->bigendian
, stride
);
253 src
+= bw
* bh
* bpp
;
256 bg
= vmnc_get_pixel(src
, bpp
, c
->bigendian
); src
+= bpp
;
259 fg
= vmnc_get_pixel(src
, bpp
, c
->bigendian
); src
+= bpp
;
264 color
= !!(flags
& HT_CLR
);
266 paint_rect(dst2
, 0, 0, bw
, bh
, bg
, bpp
, stride
);
268 if(src
- ssrc
> ssize
- rects
* (color
* bpp
+ 2)) {
269 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
272 for(k
= 0; k
< rects
; k
++) {
274 fg
= vmnc_get_pixel(src
, bpp
, c
->bigendian
); src
+= bpp
;
278 paint_rect(dst2
, xy
>> 4, xy
& 0xF, (wh
>>4)+1, (wh
& 0xF)+1, fg
, bpp
, stride
);
287 static int decode_frame(AVCodecContext
*avctx
, void *data
, int *data_size
, const uint8_t *buf
, int buf_size
)
289 VmncContext
* const c
= avctx
->priv_data
;
291 const uint8_t *src
= buf
;
292 int dx
, dy
, w
, h
, depth
, enc
, chunks
, res
, size_left
;
294 c
->pic
.reference
= 1;
295 c
->pic
.buffer_hints
= FF_BUFFER_HINTS_VALID
| FF_BUFFER_HINTS_PRESERVE
| FF_BUFFER_HINTS_REUSABLE
;
296 if(avctx
->reget_buffer(avctx
, &c
->pic
) < 0){
297 av_log(avctx
, AV_LOG_ERROR
, "reget_buffer() failed\n");
301 c
->pic
.key_frame
= 0;
302 c
->pic
.pict_type
= FF_P_TYPE
;
304 //restore screen after cursor
308 if(c
->width
< c
->cur_x
+ w
) w
= c
->width
- c
->cur_x
;
310 if(c
->height
< c
->cur_y
+ h
) h
= c
->height
- c
->cur_y
;
321 if((w
> 0) && (h
> 0)) {
322 outptr
= c
->pic
.data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
.linesize
[0];
323 for(i
= 0; i
< h
; i
++) {
324 memcpy(outptr
, c
->screendta
+ i
* c
->cur_w
* c
->bpp2
, w
* c
->bpp2
);
325 outptr
+= c
->pic
.linesize
[0];
330 chunks
= AV_RB16(src
); src
+= 2;
332 dx
= AV_RB16(src
); src
+= 2;
333 dy
= AV_RB16(src
); src
+= 2;
334 w
= AV_RB16(src
); src
+= 2;
335 h
= AV_RB16(src
); src
+= 2;
336 enc
= AV_RB32(src
); src
+= 4;
337 outptr
= c
->pic
.data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
.linesize
[0];
338 size_left
= buf_size
- (src
- buf
);
340 case MAGIC_WMVd
: // cursor
341 if(size_left
< 2 + w
* h
* c
->bpp2
* 2) {
342 av_log(avctx
, AV_LOG_ERROR
, "Premature end of data! (need %i got %i)\n", 2 + w
* h
* c
->bpp2
* 2, size_left
);
350 if((c
->cur_hx
> c
->cur_w
) || (c
->cur_hy
> c
->cur_h
)) {
351 av_log(avctx
, AV_LOG_ERROR
, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c
->cur_hx
, c
->cur_hy
, c
->cur_w
, c
->cur_h
);
352 c
->cur_hx
= c
->cur_hy
= 0;
354 c
->curbits
= av_realloc(c
->curbits
, c
->cur_w
* c
->cur_h
* c
->bpp2
);
355 c
->curmask
= av_realloc(c
->curmask
, c
->cur_w
* c
->cur_h
* c
->bpp2
);
356 c
->screendta
= av_realloc(c
->screendta
, c
->cur_w
* c
->cur_h
* c
->bpp2
);
358 src
+= w
* h
* c
->bpp2
* 2;
360 case MAGIC_WMVe
: // unknown
363 case MAGIC_WMVf
: // update cursor position
364 c
->cur_x
= dx
- c
->cur_hx
;
365 c
->cur_y
= dy
- c
->cur_hy
;
367 case MAGIC_WMVg
: // unknown
370 case MAGIC_WMVh
: // unknown
373 case MAGIC_WMVi
: // ServerInitialization struct
374 c
->pic
.key_frame
= 1;
375 c
->pic
.pict_type
= FF_I_TYPE
;
377 if(depth
!= c
->bpp
) {
378 av_log(avctx
, AV_LOG_INFO
, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c
->bpp
, depth
);
381 c
->bigendian
= *src
++;
382 if(c
->bigendian
& (~1)) {
383 av_log(avctx
, AV_LOG_INFO
, "Invalid header: bigendian flag = %i\n", c
->bigendian
);
386 //skip the rest of pixel format data
389 case MAGIC_WMVj
: // unknown
392 case 0x00000000: // raw rectangle data
393 if((dx
+ w
> c
->width
) || (dy
+ h
> c
->height
)) {
394 av_log(avctx
, AV_LOG_ERROR
, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w
, h
, dx
, dy
, c
->width
, c
->height
);
397 if(size_left
< w
* h
* c
->bpp2
) {
398 av_log(avctx
, AV_LOG_ERROR
, "Premature end of data! (need %i got %i)\n", w
* h
* c
->bpp2
, size_left
);
401 paint_raw(outptr
, w
, h
, src
, c
->bpp2
, c
->bigendian
, c
->pic
.linesize
[0]);
402 src
+= w
* h
* c
->bpp2
;
404 case 0x00000005: // HexTile encoded rectangle
405 if((dx
+ w
> c
->width
) || (dy
+ h
> c
->height
)) {
406 av_log(avctx
, AV_LOG_ERROR
, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w
, h
, dx
, dy
, c
->width
, c
->height
);
409 res
= decode_hextile(c
, outptr
, src
, size_left
, w
, h
, c
->pic
.linesize
[0]);
415 av_log(avctx
, AV_LOG_ERROR
, "Unsupported block type 0x%08X\n", enc
);
416 chunks
= 0; // leave chunks decoding loop
421 //save screen data before painting cursor
423 if(c
->width
< c
->cur_x
+ w
) w
= c
->width
- c
->cur_x
;
425 if(c
->height
< c
->cur_y
+ h
) h
= c
->height
- c
->cur_y
;
436 if((w
> 0) && (h
> 0)) {
437 outptr
= c
->pic
.data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
.linesize
[0];
438 for(i
= 0; i
< h
; i
++) {
439 memcpy(c
->screendta
+ i
* c
->cur_w
* c
->bpp2
, outptr
, w
* c
->bpp2
);
440 outptr
+= c
->pic
.linesize
[0];
442 outptr
= c
->pic
.data
[0];
443 put_cursor(outptr
, c
->pic
.linesize
[0], c
, c
->cur_x
, c
->cur_y
);
446 *data_size
= sizeof(AVFrame
);
447 *(AVFrame
*)data
= c
->pic
;
449 /* always report that the buffer was completely consumed */
460 static av_cold
int decode_init(AVCodecContext
*avctx
)
462 VmncContext
* const c
= avctx
->priv_data
;
466 c
->pic
.data
[0] = NULL
;
467 c
->width
= avctx
->width
;
468 c
->height
= avctx
->height
;
470 if (avcodec_check_dimensions(avctx
, avctx
->width
, avctx
->height
) < 0) {
473 c
->bpp
= avctx
->bits_per_coded_sample
;
478 avctx
->pix_fmt
= PIX_FMT_PAL8
;
481 avctx
->pix_fmt
= PIX_FMT_RGB555
;
484 avctx
->pix_fmt
= PIX_FMT_RGB32
;
487 av_log(avctx
, AV_LOG_ERROR
, "Unsupported bitdepth %i\n", c
->bpp
);
497 * Uninit VMnc decoder
500 static av_cold
int decode_end(AVCodecContext
*avctx
)
502 VmncContext
* const c
= avctx
->priv_data
;
505 avctx
->release_buffer(avctx
, &c
->pic
);
509 av_free(c
->screendta
);
513 AVCodec vmnc_decoder
= {
522 .long_name
= NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),