Mercurial > libavcodec.hg
annotate vmnc.c @ 3990:746a60ba3177 libavcodec
enable CMOV_IS_FAST as its faster or equal speed on every cpu (duron, athlon, PM, P3) from which ive seen benchmarks, it might be slower on P4 but noone has posted benchmarks ...
author | michael |
---|---|
date | Wed, 11 Oct 2006 12:23:40 +0000 |
parents | c8c591fe26f8 |
children | d6f83e2f8804 |
rev | line source |
---|---|
3677 | 1 /* |
2 * VMware Screen Codec (VMnc) decoder | |
3 * Copyright (c) 2006 Konstantin Shishkov | |
4 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
5 * This file is part of FFmpeg. |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
6 * |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
7 * FFmpeg is free software; you can redistribute it and/or |
3677 | 8 * modify it under the terms of the GNU Lesser General Public |
9 * License as published by the Free Software Foundation; either | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
10 * version 2.1 of the License, or (at your option) any later version. |
3677 | 11 * |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
12 * FFmpeg is distributed in the hope that it will be useful, |
3677 | 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 * Lesser General Public License for more details. | |
16 * | |
17 * You should have received a copy of the GNU Lesser General Public | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
18 * License along with FFmpeg; if not, write to the Free Software |
3677 | 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
20 * | |
21 */ | |
22 | |
23 /** | |
24 * @file vmnc.c | |
25 * VMware Screen Codec (VMnc) decoder | |
26 * As Alex Beregszaszi discovered, this is effectively RFB data dump | |
27 */ | |
28 | |
29 #include <stdio.h> | |
30 #include <stdlib.h> | |
31 | |
32 #include "common.h" | |
33 #include "avcodec.h" | |
34 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
35 enum EncTypes { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
36 MAGIC_WMVd = 0x574D5664, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
37 MAGIC_WMVe, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
38 MAGIC_WMVf, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
39 MAGIC_WMVg, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
40 MAGIC_WMVh, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
41 MAGIC_WMVi, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
42 MAGIC_WMVj |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
43 }; |
3677 | 44 |
45 enum HexTile_Flags { | |
46 HT_RAW = 1, // tile is raw | |
47 HT_BKG = 2, // background color is present | |
48 HT_FG = 4, // foreground color is present | |
49 HT_SUB = 8, // subrects are present | |
50 HT_CLR = 16 // each subrect has own color | |
51 }; | |
52 | |
53 /* | |
54 * Decoder context | |
55 */ | |
56 typedef struct VmncContext { | |
57 AVCodecContext *avctx; | |
58 AVFrame pic; | |
59 | |
60 int bpp; | |
61 int bpp2; | |
62 int bigendian; | |
63 uint8_t pal[768]; | |
64 int width, height; | |
3684 | 65 |
66 /* cursor data */ | |
67 int cur_w, cur_h; | |
68 int cur_x, cur_y; | |
69 int cur_hx, cur_hy; | |
70 uint8_t* curbits, *curmask; | |
71 uint8_t* screendta; | |
3677 | 72 } VmncContext; |
73 | |
74 /* read pixel value from stream */ | |
75 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) { | |
76 switch(bpp * 2 + be) { | |
77 case 2: | |
78 case 3: return *buf; | |
79 case 4: return LE_16(buf); | |
80 case 5: return BE_16(buf); | |
81 case 8: return LE_32(buf); | |
82 case 9: return BE_32(buf); | |
83 default: return 0; | |
84 } | |
85 } | |
86 | |
3684 | 87 static void load_cursor(VmncContext *c, uint8_t *src) |
88 { | |
89 int i, j, p; | |
90 const int bpp = c->bpp2; | |
91 uint8_t *dst8 = c->curbits; | |
92 uint16_t *dst16 = (uint16_t*)c->curbits; | |
93 uint32_t *dst32 = (uint32_t*)c->curbits; | |
94 | |
95 for(j = 0; j < c->cur_h; j++) { | |
96 for(i = 0; i < c->cur_w; i++) { | |
97 p = vmnc_get_pixel(src, bpp, c->bigendian); | |
98 src += bpp; | |
99 if(bpp == 1) *dst8++ = p; | |
100 if(bpp == 2) *dst16++ = p; | |
101 if(bpp == 4) *dst32++ = p; | |
102 } | |
103 } | |
104 dst8 = c->curmask; | |
105 dst16 = (uint16_t*)c->curmask; | |
106 dst32 = (uint32_t*)c->curmask; | |
107 for(j = 0; j < c->cur_h; j++) { | |
108 for(i = 0; i < c->cur_w; i++) { | |
109 p = vmnc_get_pixel(src, bpp, c->bigendian); | |
110 src += bpp; | |
111 if(bpp == 1) *dst8++ = p; | |
112 if(bpp == 2) *dst16++ = p; | |
113 if(bpp == 4) *dst32++ = p; | |
114 } | |
115 } | |
116 } | |
117 | |
118 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) | |
119 { | |
3920
3a151ccc6ed7
Remove unused variables and the corresponding warnings along with them.
diego
parents:
3800
diff
changeset
|
120 int i, j; |
3684 | 121 int w, h, x, y; |
122 w = c->cur_w; | |
123 if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x; | |
124 h = c->cur_h; | |
125 if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y; | |
126 x = c->cur_x; | |
127 y = c->cur_y; | |
128 if(x < 0) { | |
129 w += x; | |
130 x = 0; | |
131 } | |
132 if(y < 0) { | |
133 h += y; | |
134 y = 0; | |
135 } | |
136 | |
137 if((w < 1) || (h < 1)) return; | |
138 dst += x * c->bpp2 + y * stride; | |
139 | |
140 if(c->bpp2 == 1) { | |
141 uint8_t* cd = c->curbits, *msk = c->curmask; | |
142 for(j = 0; j < h; j++) { | |
143 for(i = 0; i < w; i++) | |
144 dst[i] = (dst[i] & cd[i]) ^ msk[i]; | |
145 msk += c->cur_w; | |
146 cd += c->cur_w; | |
147 dst += stride; | |
148 } | |
149 } else if(c->bpp2 == 2) { | |
150 uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; | |
151 uint16_t* dst2; | |
152 for(j = 0; j < h; j++) { | |
153 dst2 = (uint16_t*)dst; | |
154 for(i = 0; i < w; i++) | |
155 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
156 msk += c->cur_w; | |
157 cd += c->cur_w; | |
158 dst += stride; | |
159 } | |
160 } else if(c->bpp2 == 4) { | |
161 uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; | |
162 uint32_t* dst2; | |
163 for(j = 0; j < h; j++) { | |
164 dst2 = (uint32_t*)dst; | |
165 for(i = 0; i < w; i++) | |
166 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
167 msk += c->cur_w; | |
168 cd += c->cur_w; | |
169 dst += stride; | |
170 } | |
171 } | |
172 } | |
173 | |
3677 | 174 /* fill rectangle with given colour */ |
175 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride) | |
176 { | |
177 int i, j; | |
178 dst += dx * bpp + dy * stride; | |
179 if(bpp == 1){ | |
180 for(j = 0; j < h; j++) { | |
181 memset(dst, color, w); | |
182 dst += stride; | |
183 } | |
184 }else if(bpp == 2){ | |
185 uint16_t* dst2; | |
186 for(j = 0; j < h; j++) { | |
187 dst2 = (uint16_t*)dst; | |
188 for(i = 0; i < w; i++) { | |
189 *dst2++ = color; | |
190 } | |
191 dst += stride; | |
192 } | |
193 }else if(bpp == 4){ | |
194 uint32_t* dst2; | |
195 for(j = 0; j < h; j++) { | |
196 dst2 = (uint32_t*)dst; | |
197 for(i = 0; i < w; i++) { | |
198 dst2[i] = color; | |
199 } | |
200 dst += stride; | |
201 } | |
202 } | |
203 } | |
204 | |
205 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride) | |
206 { | |
207 int i, j, p; | |
208 for(j = 0; j < h; j++) { | |
209 for(i = 0; i < w; i++) { | |
210 p = vmnc_get_pixel(src, bpp, be); | |
211 src += bpp; | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
212 switch(bpp){ |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
213 case 1: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
214 dst[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
215 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
216 case 2: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
217 ((uint16_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
218 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
219 case 4: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
220 ((uint32_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
221 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
222 } |
3677 | 223 } |
224 dst += stride; | |
225 } | |
226 } | |
227 | |
3685 | 228 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int ssize, int w, int h, int stride) |
3677 | 229 { |
230 int i, j, k; | |
231 int bg = 0, fg = 0, rects, color, flags, xy, wh; | |
232 const int bpp = c->bpp2; | |
233 uint8_t *dst2; | |
234 int bw = 16, bh = 16; | |
235 uint8_t *ssrc=src; | |
236 | |
237 for(j = 0; j < h; j += 16) { | |
238 dst2 = dst; | |
239 bw = 16; | |
240 if(j + 16 > h) bh = h - j; | |
241 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) { | |
3685 | 242 if(src - ssrc >= ssize) { |
243 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
244 return -1; | |
245 } | |
3677 | 246 if(i + 16 > w) bw = w - i; |
247 flags = *src++; | |
248 if(flags & HT_RAW) { | |
3685 | 249 if(src - ssrc > ssize - bw * bh * bpp) { |
250 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
251 return -1; | |
252 } | |
3677 | 253 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride); |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
254 src += bw * bh * bpp; |
3677 | 255 } else { |
256 if(flags & HT_BKG) { | |
257 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
258 } | |
259 if(flags & HT_FG) { | |
260 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
261 } | |
262 rects = 0; | |
263 if(flags & HT_SUB) | |
264 rects = *src++; | |
3685 | 265 color = !!(flags & HT_CLR); |
3677 | 266 |
267 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); | |
268 | |
3685 | 269 if(src - ssrc > ssize - rects * (color * bpp + 2)) { |
270 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
271 return -1; | |
272 } | |
3677 | 273 for(k = 0; k < rects; k++) { |
274 if(color) { | |
275 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
276 } | |
277 xy = *src++; | |
278 wh = *src++; | |
279 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); | |
280 } | |
281 } | |
282 } | |
283 dst += stride * 16; | |
284 } | |
285 return src - ssrc; | |
286 } | |
287 | |
288 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size) | |
289 { | |
290 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
291 uint8_t *outptr; | |
292 uint8_t *src = buf; | |
3685 | 293 int dx, dy, w, h, depth, enc, chunks, res, size_left; |
3677 | 294 |
295 c->pic.reference = 1; | |
296 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; | |
297 if(avctx->reget_buffer(avctx, &c->pic) < 0){ | |
298 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); | |
299 return -1; | |
300 } | |
301 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
302 c->pic.key_frame = 0; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
303 c->pic.pict_type = FF_P_TYPE; |
3677 | 304 |
3684 | 305 //restore screen after cursor |
306 if(c->screendta) { | |
307 int i; | |
308 w = c->cur_w; | |
309 if(c->width < c->cur_x + w) w = c->width - c->cur_x; | |
310 h = c->cur_h; | |
311 if(c->height < c->cur_y + h) h = c->height - c->cur_y; | |
312 dx = c->cur_x; | |
313 if(dx < 0) { | |
314 w += dx; | |
315 dx = 0; | |
316 } | |
317 dy = c->cur_y; | |
318 if(dy < 0) { | |
319 h += dy; | |
320 dy = 0; | |
321 } | |
322 if((w > 0) && (h > 0)) { | |
323 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; | |
324 for(i = 0; i < h; i++) { | |
325 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2); | |
326 outptr += c->pic.linesize[0]; | |
327 } | |
328 } | |
329 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
330 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
331 chunks = BE_16(src); src += 2; |
3677 | 332 while(chunks--) { |
333 dx = BE_16(src); src += 2; | |
334 dy = BE_16(src); src += 2; | |
335 w = BE_16(src); src += 2; | |
336 h = BE_16(src); src += 2; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
337 enc = BE_32(src); src += 4; |
3684 | 338 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; |
3685 | 339 size_left = buf_size - (src - buf); |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
340 switch(enc) { |
3684 | 341 case MAGIC_WMVd: // cursor |
3685 | 342 if(size_left < 2 + w * h * c->bpp2 * 2) { |
343 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left); | |
344 return -1; | |
345 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
346 src += 2; |
3684 | 347 c->cur_w = w; |
348 c->cur_h = h; | |
349 c->cur_hx = dx; | |
350 c->cur_hy = dy; | |
351 if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { | |
352 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); | |
353 c->cur_hx = c->cur_hy = 0; | |
354 } | |
355 c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2); | |
356 c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2); | |
357 c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2); | |
358 load_cursor(c, src); | |
359 src += w * h * c->bpp2 * 2; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
360 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
361 case MAGIC_WMVe: // unknown |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
362 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
363 break; |
3684 | 364 case MAGIC_WMVf: // update cursor position |
365 c->cur_x = dx - c->cur_hx; | |
366 c->cur_y = dy - c->cur_hy; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
367 break; |
3686 | 368 case MAGIC_WMVg: // unknown |
369 src += 10; | |
370 break; | |
371 case MAGIC_WMVh: // unknown | |
372 src += 4; | |
373 break; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
374 case MAGIC_WMVi: // ServerInitialization struct |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
375 c->pic.key_frame = 1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
376 c->pic.pict_type = FF_I_TYPE; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
377 depth = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
378 if(depth != c->bpp) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
379 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
380 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
381 src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
382 c->bigendian = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
383 if(c->bigendian & (~1)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
384 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
385 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
386 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
387 //skip the rest of pixel format data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
388 src += 13; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
389 break; |
3686 | 390 case MAGIC_WMVj: // unknown |
391 src += 2; | |
392 break; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
393 case 0x00000000: // raw rectangle data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
394 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
395 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
396 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
397 } |
3685 | 398 if(size_left < w * h * c->bpp2) { |
399 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left); | |
400 return -1; | |
401 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
402 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
403 src += w * h * c->bpp2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
404 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
405 case 0x00000005: // HexTile encoded rectangle |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
406 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
407 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
408 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
409 } |
3685 | 410 res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]); |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
411 if(res < 0) |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
412 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
413 src += res; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
414 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
415 default: |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
416 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
417 chunks = 0; // leave chunks decoding loop |
3677 | 418 } |
419 } | |
3684 | 420 if(c->screendta){ |
421 int i; | |
422 //save screen data before painting cursor | |
423 w = c->cur_w; | |
424 if(c->width < c->cur_x + w) w = c->width - c->cur_x; | |
425 h = c->cur_h; | |
426 if(c->height < c->cur_y + h) h = c->height - c->cur_y; | |
427 dx = c->cur_x; | |
428 if(dx < 0) { | |
429 w += dx; | |
430 dx = 0; | |
431 } | |
432 dy = c->cur_y; | |
433 if(dy < 0) { | |
434 h += dy; | |
435 dy = 0; | |
436 } | |
437 if((w > 0) && (h > 0)) { | |
438 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; | |
439 for(i = 0; i < h; i++) { | |
440 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2); | |
441 outptr += c->pic.linesize[0]; | |
442 } | |
443 outptr = c->pic.data[0]; | |
444 put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y); | |
445 } | |
446 } | |
3677 | 447 *data_size = sizeof(AVFrame); |
448 *(AVFrame*)data = c->pic; | |
449 | |
450 /* always report that the buffer was completely consumed */ | |
451 return buf_size; | |
452 } | |
453 | |
454 | |
455 | |
456 /* | |
457 * | |
458 * Init VMnc decoder | |
459 * | |
460 */ | |
461 static int decode_init(AVCodecContext *avctx) | |
462 { | |
463 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
464 | |
465 c->avctx = avctx; | |
466 avctx->has_b_frames = 0; | |
467 | |
468 c->pic.data[0] = NULL; | |
469 c->width = avctx->width; | |
470 c->height = avctx->height; | |
471 | |
3800
9b75ab171fa9
1l: correct argument order in avcodec_check_dimensions
kostya
parents:
3686
diff
changeset
|
472 if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) { |
3677 | 473 return 1; |
474 } | |
475 c->bpp = avctx->bits_per_sample; | |
476 c->bpp2 = c->bpp/8; | |
477 | |
478 switch(c->bpp){ | |
479 case 8: | |
480 avctx->pix_fmt = PIX_FMT_PAL8; | |
481 break; | |
482 case 16: | |
483 avctx->pix_fmt = PIX_FMT_RGB555; | |
484 break; | |
485 case 32: | |
486 avctx->pix_fmt = PIX_FMT_RGB32; | |
487 break; | |
488 default: | |
489 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); | |
490 } | |
491 | |
492 return 0; | |
493 } | |
494 | |
495 | |
496 | |
497 /* | |
498 * | |
499 * Uninit VMnc decoder | |
500 * | |
501 */ | |
502 static int decode_end(AVCodecContext *avctx) | |
503 { | |
504 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
505 | |
506 if (c->pic.data[0]) | |
507 avctx->release_buffer(avctx, &c->pic); | |
508 | |
3684 | 509 av_free(c->curbits); |
510 av_free(c->curmask); | |
511 av_free(c->screendta); | |
3677 | 512 return 0; |
513 } | |
514 | |
515 AVCodec vmnc_decoder = { | |
516 "VMware video", | |
517 CODEC_TYPE_VIDEO, | |
518 CODEC_ID_VMNC, | |
519 sizeof(VmncContext), | |
520 decode_init, | |
521 NULL, | |
522 decode_end, | |
523 decode_frame | |
524 }; | |
525 |