Mercurial > libavcodec.hg
annotate vmnc.c @ 3678:93a961e40d65 libavcodec
Handle raw blocks correctly (both updating pointer and storing to memory)
author | kostya |
---|---|
date | Tue, 05 Sep 2006 07:29:26 +0000 |
parents | 18b13b923616 |
children | 2702eec8c8b9 |
rev | line source |
---|---|
3677 | 1 /* |
2 * VMware Screen Codec (VMnc) decoder | |
3 * Copyright (c) 2006 Konstantin Shishkov | |
4 * | |
5 * This library is free software; you can redistribute it and/or | |
6 * modify it under the terms of the GNU Lesser General Public | |
7 * License as published by the Free Software Foundation; either | |
8 * version 2 of the License, or (at your option) any later version. | |
9 * | |
10 * This library is distributed in the hope that it will be useful, | |
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Lesser General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Lesser General Public | |
16 * License along with this library; if not, write to the Free Software | |
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
18 * | |
19 */ | |
20 | |
21 /** | |
22 * @file vmnc.c | |
23 * VMware Screen Codec (VMnc) decoder | |
24 * As Alex Beregszaszi discovered, this is effectively RFB data dump | |
25 */ | |
26 | |
27 #include <stdio.h> | |
28 #include <stdlib.h> | |
29 | |
30 #include "common.h" | |
31 #include "avcodec.h" | |
32 | |
33 #define MAGIC_WMVi 0x574D5669 | |
34 | |
35 enum HexTile_Flags { | |
36 HT_RAW = 1, // tile is raw | |
37 HT_BKG = 2, // background color is present | |
38 HT_FG = 4, // foreground color is present | |
39 HT_SUB = 8, // subrects are present | |
40 HT_CLR = 16 // each subrect has own color | |
41 }; | |
42 | |
43 /* | |
44 * Decoder context | |
45 */ | |
46 typedef struct VmncContext { | |
47 AVCodecContext *avctx; | |
48 AVFrame pic; | |
49 | |
50 int bpp; | |
51 int bpp2; | |
52 int bigendian; | |
53 uint8_t pal[768]; | |
54 int width, height; | |
55 } VmncContext; | |
56 | |
57 /* read pixel value from stream */ | |
58 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) { | |
59 switch(bpp * 2 + be) { | |
60 case 2: | |
61 case 3: return *buf; | |
62 case 4: return LE_16(buf); | |
63 case 5: return BE_16(buf); | |
64 case 8: return LE_32(buf); | |
65 case 9: return BE_32(buf); | |
66 default: return 0; | |
67 } | |
68 } | |
69 | |
70 /* fill rectangle with given colour */ | |
71 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride) | |
72 { | |
73 int i, j; | |
74 dst += dx * bpp + dy * stride; | |
75 if(bpp == 1){ | |
76 for(j = 0; j < h; j++) { | |
77 memset(dst, color, w); | |
78 dst += stride; | |
79 } | |
80 }else if(bpp == 2){ | |
81 uint16_t* dst2; | |
82 for(j = 0; j < h; j++) { | |
83 dst2 = (uint16_t*)dst; | |
84 for(i = 0; i < w; i++) { | |
85 *dst2++ = color; | |
86 } | |
87 dst += stride; | |
88 } | |
89 }else if(bpp == 4){ | |
90 uint32_t* dst2; | |
91 for(j = 0; j < h; j++) { | |
92 dst2 = (uint32_t*)dst; | |
93 for(i = 0; i < w; i++) { | |
94 dst2[i] = color; | |
95 } | |
96 dst += stride; | |
97 } | |
98 } | |
99 } | |
100 | |
101 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride) | |
102 { | |
103 int i, j, p; | |
104 for(j = 0; j < h; j++) { | |
105 for(i = 0; i < w; i++) { | |
106 p = vmnc_get_pixel(src, bpp, be); | |
107 src += bpp; | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
108 switch(bpp){ |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
109 case 1: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
110 dst[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
111 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
112 case 2: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
113 ((uint16_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
114 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
115 case 4: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
116 ((uint32_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
117 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
118 } |
3677 | 119 } |
120 dst += stride; | |
121 } | |
122 } | |
123 | |
124 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int w, int h, int stride) | |
125 { | |
126 int i, j, k; | |
127 int bg = 0, fg = 0, rects, color, flags, xy, wh; | |
128 const int bpp = c->bpp2; | |
129 uint8_t *dst2; | |
130 int bw = 16, bh = 16; | |
131 uint8_t *ssrc=src; | |
132 | |
133 for(j = 0; j < h; j += 16) { | |
134 dst2 = dst; | |
135 bw = 16; | |
136 if(j + 16 > h) bh = h - j; | |
137 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) { | |
138 if(i + 16 > w) bw = w - i; | |
139 flags = *src++; | |
140 if(flags & HT_RAW) { | |
141 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride); | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
142 src += bw * bh * bpp; |
3677 | 143 } else { |
144 if(flags & HT_BKG) { | |
145 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
146 } | |
147 if(flags & HT_FG) { | |
148 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
149 } | |
150 rects = 0; | |
151 if(flags & HT_SUB) | |
152 rects = *src++; | |
153 color = (flags & HT_CLR); | |
154 | |
155 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); | |
156 | |
157 for(k = 0; k < rects; k++) { | |
158 if(color) { | |
159 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
160 } | |
161 xy = *src++; | |
162 wh = *src++; | |
163 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); | |
164 } | |
165 } | |
166 } | |
167 dst += stride * 16; | |
168 } | |
169 return src - ssrc; | |
170 } | |
171 | |
172 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size) | |
173 { | |
174 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
175 uint8_t *outptr; | |
176 uint8_t *src = buf; | |
177 int t, dx, dy, w, h, enc, chunks, res; | |
178 | |
179 c->pic.reference = 1; | |
180 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; | |
181 if(avctx->reget_buffer(avctx, &c->pic) < 0){ | |
182 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); | |
183 return -1; | |
184 } | |
185 | |
186 t = BE_32(src); | |
187 src += 4; | |
188 | |
189 chunks = t & 0xFF; | |
190 if(chunks > 8) { | |
191 av_log(avctx, AV_LOG_ERROR, "Frame decoding is not possible. Please report sample to developers.\n"); | |
192 return -1; | |
193 } | |
194 if(chunks == 8) { | |
195 int w, h, depth; | |
196 c->pic.key_frame = 1; | |
197 c->pic.pict_type = FF_I_TYPE; | |
198 | |
199 /* parse ServerInitialization struct */ | |
200 src += 4; | |
201 w = BE_16(src); src += 2; | |
202 h = BE_16(src); src += 2; | |
203 t = BE_32(src); src += 4; | |
204 if(t != MAGIC_WMVi) { | |
205 av_log(avctx, AV_LOG_INFO, "Invalid header: magic not found\n"); | |
206 return -1; | |
207 } | |
208 depth = *src++; | |
209 if(depth != c->bpp) { | |
210 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth); | |
211 } | |
212 src++; | |
213 c->bigendian = *src++; | |
214 if(c->bigendian & (~1)) { | |
215 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian); | |
216 return -1; | |
217 } | |
218 //skip pixel format data | |
219 src += 13; | |
220 chunks = 1; // there should be one chunk with the whole frame, rest could be ignored | |
221 } else { | |
222 c->pic.key_frame = 0; | |
223 c->pic.pict_type = FF_P_TYPE; | |
224 } | |
225 while(chunks--) { | |
226 // decode FramebufferUpdate struct | |
227 dx = BE_16(src); src += 2; | |
228 dy = BE_16(src); src += 2; | |
229 w = BE_16(src); src += 2; | |
230 h = BE_16(src); src += 2; | |
231 if((dx + w > c->width) || (dy + h > c->height)) { | |
232 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); | |
233 return -1; | |
234 } | |
235 enc = BE_32(src); src += 4; | |
236 if(enc != 0x00000005) { | |
237 av_log(avctx, AV_LOG_ERROR, "Only hextile decoding is supported for now\n"); | |
238 switch(enc) { | |
239 case 0: | |
240 av_log(avctx, AV_LOG_INFO, "And this is raw encoding\n"); | |
241 break; | |
242 case 1: | |
243 av_log(avctx, AV_LOG_INFO, "And this is CopyRect encoding\n"); | |
244 break; | |
245 case 2: | |
246 av_log(avctx, AV_LOG_INFO, "And this is RRE encoding\n"); | |
247 break; | |
248 case 3: | |
249 av_log(avctx, AV_LOG_INFO, "And this is CoRRE encoding\n"); | |
250 break; | |
251 default: | |
252 av_log(avctx, AV_LOG_INFO, "And this is unknown encoding (%i)\n", enc); | |
253 } | |
254 return -1; | |
255 } | |
256 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; | |
257 res = decode_hextile(c, outptr, src, w, h, c->pic.linesize[0]); | |
258 if(res < 0) | |
259 return -1; | |
260 src += res; | |
261 } | |
262 *data_size = sizeof(AVFrame); | |
263 *(AVFrame*)data = c->pic; | |
264 | |
265 /* always report that the buffer was completely consumed */ | |
266 return buf_size; | |
267 } | |
268 | |
269 | |
270 | |
271 /* | |
272 * | |
273 * Init VMnc decoder | |
274 * | |
275 */ | |
276 static int decode_init(AVCodecContext *avctx) | |
277 { | |
278 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
279 | |
280 c->avctx = avctx; | |
281 avctx->has_b_frames = 0; | |
282 | |
283 c->pic.data[0] = NULL; | |
284 c->width = avctx->width; | |
285 c->height = avctx->height; | |
286 | |
287 if (avcodec_check_dimensions(avctx, avctx->height, avctx->width) < 0) { | |
288 return 1; | |
289 } | |
290 c->bpp = avctx->bits_per_sample; | |
291 c->bpp2 = c->bpp/8; | |
292 | |
293 switch(c->bpp){ | |
294 case 8: | |
295 avctx->pix_fmt = PIX_FMT_PAL8; | |
296 break; | |
297 case 16: | |
298 avctx->pix_fmt = PIX_FMT_RGB555; | |
299 break; | |
300 case 32: | |
301 avctx->pix_fmt = PIX_FMT_RGB32; | |
302 break; | |
303 default: | |
304 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); | |
305 } | |
306 | |
307 return 0; | |
308 } | |
309 | |
310 | |
311 | |
312 /* | |
313 * | |
314 * Uninit VMnc decoder | |
315 * | |
316 */ | |
317 static int decode_end(AVCodecContext *avctx) | |
318 { | |
319 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
320 | |
321 if (c->pic.data[0]) | |
322 avctx->release_buffer(avctx, &c->pic); | |
323 | |
324 return 0; | |
325 } | |
326 | |
327 AVCodec vmnc_decoder = { | |
328 "VMware video", | |
329 CODEC_TYPE_VIDEO, | |
330 CODEC_ID_VMNC, | |
331 sizeof(VmncContext), | |
332 decode_init, | |
333 NULL, | |
334 decode_end, | |
335 decode_frame | |
336 }; | |
337 |