3677
|
1 /*
|
|
2 * VMware Screen Codec (VMnc) decoder
|
|
3 * Copyright (c) 2006 Konstantin Shishkov
|
|
4 *
|
|
5 * This library is free software; you can redistribute it and/or
|
|
6 * modify it under the terms of the GNU Lesser General Public
|
|
7 * License as published by the Free Software Foundation; either
|
|
8 * version 2 of the License, or (at your option) any later version.
|
|
9 *
|
|
10 * This library is distributed in the hope that it will be useful,
|
|
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
13 * Lesser General Public License for more details.
|
|
14 *
|
|
15 * You should have received a copy of the GNU Lesser General Public
|
|
16 * License along with this library; if not, write to the Free Software
|
|
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
18 *
|
|
19 */
|
|
20
|
|
21 /**
|
|
22 * @file vmnc.c
|
|
23 * VMware Screen Codec (VMnc) decoder
|
|
24 * As Alex Beregszaszi discovered, this is effectively RFB data dump
|
|
25 */
|
|
26
|
|
27 #include <stdio.h>
|
|
28 #include <stdlib.h>
|
|
29
|
|
30 #include "common.h"
|
|
31 #include "avcodec.h"
|
|
32
|
|
33 #define MAGIC_WMVi 0x574D5669
|
|
34
|
|
35 enum HexTile_Flags {
|
|
36 HT_RAW = 1, // tile is raw
|
|
37 HT_BKG = 2, // background color is present
|
|
38 HT_FG = 4, // foreground color is present
|
|
39 HT_SUB = 8, // subrects are present
|
|
40 HT_CLR = 16 // each subrect has own color
|
|
41 };
|
|
42
|
|
43 /*
|
|
44 * Decoder context
|
|
45 */
|
|
46 typedef struct VmncContext {
|
|
47 AVCodecContext *avctx;
|
|
48 AVFrame pic;
|
|
49
|
|
50 int bpp;
|
|
51 int bpp2;
|
|
52 int bigendian;
|
|
53 uint8_t pal[768];
|
|
54 int width, height;
|
|
55 } VmncContext;
|
|
56
|
|
57 /* read pixel value from stream */
|
|
58 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) {
|
|
59 switch(bpp * 2 + be) {
|
|
60 case 2:
|
|
61 case 3: return *buf;
|
|
62 case 4: return LE_16(buf);
|
|
63 case 5: return BE_16(buf);
|
|
64 case 8: return LE_32(buf);
|
|
65 case 9: return BE_32(buf);
|
|
66 default: return 0;
|
|
67 }
|
|
68 }
|
|
69
|
|
70 /* fill rectangle with given colour */
|
|
71 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
|
|
72 {
|
|
73 int i, j;
|
|
74 dst += dx * bpp + dy * stride;
|
|
75 if(bpp == 1){
|
|
76 for(j = 0; j < h; j++) {
|
|
77 memset(dst, color, w);
|
|
78 dst += stride;
|
|
79 }
|
|
80 }else if(bpp == 2){
|
|
81 uint16_t* dst2;
|
|
82 for(j = 0; j < h; j++) {
|
|
83 dst2 = (uint16_t*)dst;
|
|
84 for(i = 0; i < w; i++) {
|
|
85 *dst2++ = color;
|
|
86 }
|
|
87 dst += stride;
|
|
88 }
|
|
89 }else if(bpp == 4){
|
|
90 uint32_t* dst2;
|
|
91 for(j = 0; j < h; j++) {
|
|
92 dst2 = (uint32_t*)dst;
|
|
93 for(i = 0; i < w; i++) {
|
|
94 dst2[i] = color;
|
|
95 }
|
|
96 dst += stride;
|
|
97 }
|
|
98 }
|
|
99 }
|
|
100
|
|
101 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride)
|
|
102 {
|
|
103 int i, j, p;
|
|
104 for(j = 0; j < h; j++) {
|
|
105 for(i = 0; i < w; i++) {
|
|
106 p = vmnc_get_pixel(src, bpp, be);
|
|
107 src += bpp;
|
|
108 memcpy(dst + i*bpp, &p, bpp);
|
|
109 }
|
|
110 dst += stride;
|
|
111 }
|
|
112 }
|
|
113
|
|
114 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int w, int h, int stride)
|
|
115 {
|
|
116 int i, j, k;
|
|
117 int bg = 0, fg = 0, rects, color, flags, xy, wh;
|
|
118 const int bpp = c->bpp2;
|
|
119 uint8_t *dst2;
|
|
120 int bw = 16, bh = 16;
|
|
121 uint8_t *ssrc=src;
|
|
122
|
|
123 for(j = 0; j < h; j += 16) {
|
|
124 dst2 = dst;
|
|
125 bw = 16;
|
|
126 if(j + 16 > h) bh = h - j;
|
|
127 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
|
|
128 if(i + 16 > w) bw = w - i;
|
|
129 flags = *src++;
|
|
130 if(flags & HT_RAW) {
|
|
131 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
|
|
132 } else {
|
|
133 if(flags & HT_BKG) {
|
|
134 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
|
|
135 }
|
|
136 if(flags & HT_FG) {
|
|
137 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
|
|
138 }
|
|
139 rects = 0;
|
|
140 if(flags & HT_SUB)
|
|
141 rects = *src++;
|
|
142 color = (flags & HT_CLR);
|
|
143
|
|
144 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
|
|
145
|
|
146 for(k = 0; k < rects; k++) {
|
|
147 if(color) {
|
|
148 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
|
|
149 }
|
|
150 xy = *src++;
|
|
151 wh = *src++;
|
|
152 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
|
|
153 }
|
|
154 }
|
|
155 }
|
|
156 dst += stride * 16;
|
|
157 }
|
|
158 return src - ssrc;
|
|
159 }
|
|
160
|
|
161 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size)
|
|
162 {
|
|
163 VmncContext * const c = (VmncContext *)avctx->priv_data;
|
|
164 uint8_t *outptr;
|
|
165 uint8_t *src = buf;
|
|
166 int t, dx, dy, w, h, enc, chunks, res;
|
|
167
|
|
168 c->pic.reference = 1;
|
|
169 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
|
|
170 if(avctx->reget_buffer(avctx, &c->pic) < 0){
|
|
171 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
|
|
172 return -1;
|
|
173 }
|
|
174
|
|
175 t = BE_32(src);
|
|
176 src += 4;
|
|
177
|
|
178 chunks = t & 0xFF;
|
|
179 if(chunks > 8) {
|
|
180 av_log(avctx, AV_LOG_ERROR, "Frame decoding is not possible. Please report sample to developers.\n");
|
|
181 return -1;
|
|
182 }
|
|
183 if(chunks == 8) {
|
|
184 int w, h, depth;
|
|
185 c->pic.key_frame = 1;
|
|
186 c->pic.pict_type = FF_I_TYPE;
|
|
187
|
|
188 /* parse ServerInitialization struct */
|
|
189 src += 4;
|
|
190 w = BE_16(src); src += 2;
|
|
191 h = BE_16(src); src += 2;
|
|
192 t = BE_32(src); src += 4;
|
|
193 if(t != MAGIC_WMVi) {
|
|
194 av_log(avctx, AV_LOG_INFO, "Invalid header: magic not found\n");
|
|
195 return -1;
|
|
196 }
|
|
197 depth = *src++;
|
|
198 if(depth != c->bpp) {
|
|
199 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
|
|
200 }
|
|
201 src++;
|
|
202 c->bigendian = *src++;
|
|
203 if(c->bigendian & (~1)) {
|
|
204 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
|
|
205 return -1;
|
|
206 }
|
|
207 //skip pixel format data
|
|
208 src += 13;
|
|
209 chunks = 1; // there should be one chunk with the whole frame, rest could be ignored
|
|
210 } else {
|
|
211 c->pic.key_frame = 0;
|
|
212 c->pic.pict_type = FF_P_TYPE;
|
|
213 }
|
|
214 while(chunks--) {
|
|
215 // decode FramebufferUpdate struct
|
|
216 dx = BE_16(src); src += 2;
|
|
217 dy = BE_16(src); src += 2;
|
|
218 w = BE_16(src); src += 2;
|
|
219 h = BE_16(src); src += 2;
|
|
220 if((dx + w > c->width) || (dy + h > c->height)) {
|
|
221 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
|
|
222 return -1;
|
|
223 }
|
|
224 enc = BE_32(src); src += 4;
|
|
225 if(enc != 0x00000005) {
|
|
226 av_log(avctx, AV_LOG_ERROR, "Only hextile decoding is supported for now\n");
|
|
227 switch(enc) {
|
|
228 case 0:
|
|
229 av_log(avctx, AV_LOG_INFO, "And this is raw encoding\n");
|
|
230 break;
|
|
231 case 1:
|
|
232 av_log(avctx, AV_LOG_INFO, "And this is CopyRect encoding\n");
|
|
233 break;
|
|
234 case 2:
|
|
235 av_log(avctx, AV_LOG_INFO, "And this is RRE encoding\n");
|
|
236 break;
|
|
237 case 3:
|
|
238 av_log(avctx, AV_LOG_INFO, "And this is CoRRE encoding\n");
|
|
239 break;
|
|
240 default:
|
|
241 av_log(avctx, AV_LOG_INFO, "And this is unknown encoding (%i)\n", enc);
|
|
242 }
|
|
243 return -1;
|
|
244 }
|
|
245 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
|
|
246 res = decode_hextile(c, outptr, src, w, h, c->pic.linesize[0]);
|
|
247 if(res < 0)
|
|
248 return -1;
|
|
249 src += res;
|
|
250 }
|
|
251 *data_size = sizeof(AVFrame);
|
|
252 *(AVFrame*)data = c->pic;
|
|
253
|
|
254 /* always report that the buffer was completely consumed */
|
|
255 return buf_size;
|
|
256 }
|
|
257
|
|
258
|
|
259
|
|
260 /*
|
|
261 *
|
|
262 * Init VMnc decoder
|
|
263 *
|
|
264 */
|
|
265 static int decode_init(AVCodecContext *avctx)
|
|
266 {
|
|
267 VmncContext * const c = (VmncContext *)avctx->priv_data;
|
|
268
|
|
269 c->avctx = avctx;
|
|
270 avctx->has_b_frames = 0;
|
|
271
|
|
272 c->pic.data[0] = NULL;
|
|
273 c->width = avctx->width;
|
|
274 c->height = avctx->height;
|
|
275
|
|
276 if (avcodec_check_dimensions(avctx, avctx->height, avctx->width) < 0) {
|
|
277 return 1;
|
|
278 }
|
|
279 c->bpp = avctx->bits_per_sample;
|
|
280 c->bpp2 = c->bpp/8;
|
|
281
|
|
282 switch(c->bpp){
|
|
283 case 8:
|
|
284 avctx->pix_fmt = PIX_FMT_PAL8;
|
|
285 break;
|
|
286 case 16:
|
|
287 avctx->pix_fmt = PIX_FMT_RGB555;
|
|
288 break;
|
|
289 case 32:
|
|
290 avctx->pix_fmt = PIX_FMT_RGB32;
|
|
291 break;
|
|
292 default:
|
|
293 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
|
|
294 }
|
|
295
|
|
296 return 0;
|
|
297 }
|
|
298
|
|
299
|
|
300
|
|
301 /*
|
|
302 *
|
|
303 * Uninit VMnc decoder
|
|
304 *
|
|
305 */
|
|
306 static int decode_end(AVCodecContext *avctx)
|
|
307 {
|
|
308 VmncContext * const c = (VmncContext *)avctx->priv_data;
|
|
309
|
|
310 if (c->pic.data[0])
|
|
311 avctx->release_buffer(avctx, &c->pic);
|
|
312
|
|
313 return 0;
|
|
314 }
|
|
315
|
|
316 AVCodec vmnc_decoder = {
|
|
317 "VMware video",
|
|
318 CODEC_TYPE_VIDEO,
|
|
319 CODEC_ID_VMNC,
|
|
320 sizeof(VmncContext),
|
|
321 decode_init,
|
|
322 NULL,
|
|
323 decode_end,
|
|
324 decode_frame
|
|
325 };
|
|
326
|