Mercurial > libavcodec.hg
annotate vmnc.c @ 3679:2702eec8c8b9 libavcodec
Try to handle all chunks, previous scheme was not correct.
author | kostya |
---|---|
date | Tue, 05 Sep 2006 07:31:53 +0000 |
parents | 93a961e40d65 |
children | 9f77f4f577b4 |
rev | line source |
---|---|
3677 | 1 /* |
2 * VMware Screen Codec (VMnc) decoder | |
3 * Copyright (c) 2006 Konstantin Shishkov | |
4 * | |
5 * This library is free software; you can redistribute it and/or | |
6 * modify it under the terms of the GNU Lesser General Public | |
7 * License as published by the Free Software Foundation; either | |
8 * version 2 of the License, or (at your option) any later version. | |
9 * | |
10 * This library is distributed in the hope that it will be useful, | |
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Lesser General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Lesser General Public | |
16 * License along with this library; if not, write to the Free Software | |
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
18 * | |
19 */ | |
20 | |
21 /** | |
22 * @file vmnc.c | |
23 * VMware Screen Codec (VMnc) decoder | |
24 * As Alex Beregszaszi discovered, this is effectively RFB data dump | |
25 */ | |
26 | |
27 #include <stdio.h> | |
28 #include <stdlib.h> | |
29 | |
30 #include "common.h" | |
31 #include "avcodec.h" | |
32 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
33 enum EncTypes { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
34 MAGIC_WMVd = 0x574D5664, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
35 MAGIC_WMVe, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
36 MAGIC_WMVf, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
37 MAGIC_WMVg, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
38 MAGIC_WMVh, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
39 MAGIC_WMVi, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
40 MAGIC_WMVj |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
41 }; |
3677 | 42 |
43 enum HexTile_Flags { | |
44 HT_RAW = 1, // tile is raw | |
45 HT_BKG = 2, // background color is present | |
46 HT_FG = 4, // foreground color is present | |
47 HT_SUB = 8, // subrects are present | |
48 HT_CLR = 16 // each subrect has own color | |
49 }; | |
50 | |
51 /* | |
52 * Decoder context | |
53 */ | |
54 typedef struct VmncContext { | |
55 AVCodecContext *avctx; | |
56 AVFrame pic; | |
57 | |
58 int bpp; | |
59 int bpp2; | |
60 int bigendian; | |
61 uint8_t pal[768]; | |
62 int width, height; | |
63 } VmncContext; | |
64 | |
65 /* read pixel value from stream */ | |
66 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) { | |
67 switch(bpp * 2 + be) { | |
68 case 2: | |
69 case 3: return *buf; | |
70 case 4: return LE_16(buf); | |
71 case 5: return BE_16(buf); | |
72 case 8: return LE_32(buf); | |
73 case 9: return BE_32(buf); | |
74 default: return 0; | |
75 } | |
76 } | |
77 | |
78 /* fill rectangle with given colour */ | |
79 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride) | |
80 { | |
81 int i, j; | |
82 dst += dx * bpp + dy * stride; | |
83 if(bpp == 1){ | |
84 for(j = 0; j < h; j++) { | |
85 memset(dst, color, w); | |
86 dst += stride; | |
87 } | |
88 }else if(bpp == 2){ | |
89 uint16_t* dst2; | |
90 for(j = 0; j < h; j++) { | |
91 dst2 = (uint16_t*)dst; | |
92 for(i = 0; i < w; i++) { | |
93 *dst2++ = color; | |
94 } | |
95 dst += stride; | |
96 } | |
97 }else if(bpp == 4){ | |
98 uint32_t* dst2; | |
99 for(j = 0; j < h; j++) { | |
100 dst2 = (uint32_t*)dst; | |
101 for(i = 0; i < w; i++) { | |
102 dst2[i] = color; | |
103 } | |
104 dst += stride; | |
105 } | |
106 } | |
107 } | |
108 | |
109 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride) | |
110 { | |
111 int i, j, p; | |
112 for(j = 0; j < h; j++) { | |
113 for(i = 0; i < w; i++) { | |
114 p = vmnc_get_pixel(src, bpp, be); | |
115 src += bpp; | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
116 switch(bpp){ |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
117 case 1: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
118 dst[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
119 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
120 case 2: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
121 ((uint16_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
122 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
123 case 4: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
124 ((uint32_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
125 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
126 } |
3677 | 127 } |
128 dst += stride; | |
129 } | |
130 } | |
131 | |
132 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int w, int h, int stride) | |
133 { | |
134 int i, j, k; | |
135 int bg = 0, fg = 0, rects, color, flags, xy, wh; | |
136 const int bpp = c->bpp2; | |
137 uint8_t *dst2; | |
138 int bw = 16, bh = 16; | |
139 uint8_t *ssrc=src; | |
140 | |
141 for(j = 0; j < h; j += 16) { | |
142 dst2 = dst; | |
143 bw = 16; | |
144 if(j + 16 > h) bh = h - j; | |
145 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) { | |
146 if(i + 16 > w) bw = w - i; | |
147 flags = *src++; | |
148 if(flags & HT_RAW) { | |
149 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride); | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
150 src += bw * bh * bpp; |
3677 | 151 } else { |
152 if(flags & HT_BKG) { | |
153 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
154 } | |
155 if(flags & HT_FG) { | |
156 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
157 } | |
158 rects = 0; | |
159 if(flags & HT_SUB) | |
160 rects = *src++; | |
161 color = (flags & HT_CLR); | |
162 | |
163 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); | |
164 | |
165 for(k = 0; k < rects; k++) { | |
166 if(color) { | |
167 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
168 } | |
169 xy = *src++; | |
170 wh = *src++; | |
171 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); | |
172 } | |
173 } | |
174 } | |
175 dst += stride * 16; | |
176 } | |
177 return src - ssrc; | |
178 } | |
179 | |
180 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size) | |
181 { | |
182 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
183 uint8_t *outptr; | |
184 uint8_t *src = buf; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
185 int dx, dy, w, h, depth, enc, chunks, res; |
3677 | 186 |
187 c->pic.reference = 1; | |
188 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; | |
189 if(avctx->reget_buffer(avctx, &c->pic) < 0){ | |
190 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); | |
191 return -1; | |
192 } | |
193 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
194 c->pic.key_frame = 0; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
195 c->pic.pict_type = FF_P_TYPE; |
3677 | 196 |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
197 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
198 chunks = BE_16(src); src += 2; |
3677 | 199 while(chunks--) { |
200 dx = BE_16(src); src += 2; | |
201 dy = BE_16(src); src += 2; | |
202 w = BE_16(src); src += 2; | |
203 h = BE_16(src); src += 2; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
204 enc = BE_32(src); src += 4; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
205 switch(enc) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
206 case MAGIC_WMVd: // unknown |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
207 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
208 src += w * h * 8; // skip this data for now |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
209 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
210 case MAGIC_WMVe: // unknown |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
211 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
212 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
213 case MAGIC_WMVf: // unknown and empty |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
214 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
215 case MAGIC_WMVi: // ServerInitialization struct |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
216 c->pic.key_frame = 1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
217 c->pic.pict_type = FF_I_TYPE; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
218 depth = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
219 if(depth != c->bpp) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
220 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
221 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
222 src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
223 c->bigendian = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
224 if(c->bigendian & (~1)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
225 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
226 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
227 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
228 //skip the rest of pixel format data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
229 src += 13; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
230 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
231 case 0x00000000: // raw rectangle data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
232 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
233 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
234 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
235 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
236 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
237 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
238 src += w * h * c->bpp2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
239 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
240 case 0x00000005: // HexTile encoded rectangle |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
241 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
242 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
243 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
244 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
245 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
246 res = decode_hextile(c, outptr, src, w, h, c->pic.linesize[0]); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
247 if(res < 0) |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
248 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
249 src += res; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
250 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
251 default: |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
252 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
253 chunks = 0; // leave chunks decoding loop |
3677 | 254 } |
255 } | |
256 *data_size = sizeof(AVFrame); | |
257 *(AVFrame*)data = c->pic; | |
258 | |
259 /* always report that the buffer was completely consumed */ | |
260 return buf_size; | |
261 } | |
262 | |
263 | |
264 | |
265 /* | |
266 * | |
267 * Init VMnc decoder | |
268 * | |
269 */ | |
270 static int decode_init(AVCodecContext *avctx) | |
271 { | |
272 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
273 | |
274 c->avctx = avctx; | |
275 avctx->has_b_frames = 0; | |
276 | |
277 c->pic.data[0] = NULL; | |
278 c->width = avctx->width; | |
279 c->height = avctx->height; | |
280 | |
281 if (avcodec_check_dimensions(avctx, avctx->height, avctx->width) < 0) { | |
282 return 1; | |
283 } | |
284 c->bpp = avctx->bits_per_sample; | |
285 c->bpp2 = c->bpp/8; | |
286 | |
287 switch(c->bpp){ | |
288 case 8: | |
289 avctx->pix_fmt = PIX_FMT_PAL8; | |
290 break; | |
291 case 16: | |
292 avctx->pix_fmt = PIX_FMT_RGB555; | |
293 break; | |
294 case 32: | |
295 avctx->pix_fmt = PIX_FMT_RGB32; | |
296 break; | |
297 default: | |
298 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); | |
299 } | |
300 | |
301 return 0; | |
302 } | |
303 | |
304 | |
305 | |
306 /* | |
307 * | |
308 * Uninit VMnc decoder | |
309 * | |
310 */ | |
311 static int decode_end(AVCodecContext *avctx) | |
312 { | |
313 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
314 | |
315 if (c->pic.data[0]) | |
316 avctx->release_buffer(avctx, &c->pic); | |
317 | |
318 return 0; | |
319 } | |
320 | |
321 AVCodec vmnc_decoder = { | |
322 "VMware video", | |
323 CODEC_TYPE_VIDEO, | |
324 CODEC_ID_VMNC, | |
325 sizeof(VmncContext), | |
326 decode_init, | |
327 NULL, | |
328 decode_end, | |
329 decode_frame | |
330 }; | |
331 |