Mercurial > libavcodec.hg
annotate vmnc.c @ 4560:a798e8cc5e67 libavcodec
gcc 2.95 fix
author | michael |
---|---|
date | Thu, 22 Feb 2007 00:04:36 +0000 |
parents | 05e932ddaaa9 |
children | 59f4fb490fa7 |
rev | line source |
---|---|
3677 | 1 /* |
2 * VMware Screen Codec (VMnc) decoder | |
3 * Copyright (c) 2006 Konstantin Shishkov | |
4 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
5 * This file is part of FFmpeg. |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
6 * |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
7 * FFmpeg is free software; you can redistribute it and/or |
3677 | 8 * modify it under the terms of the GNU Lesser General Public |
9 * License as published by the Free Software Foundation; either | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
10 * version 2.1 of the License, or (at your option) any later version. |
3677 | 11 * |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
12 * FFmpeg is distributed in the hope that it will be useful, |
3677 | 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 * Lesser General Public License for more details. | |
16 * | |
17 * You should have received a copy of the GNU Lesser General Public | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3920
diff
changeset
|
18 * License along with FFmpeg; if not, write to the Free Software |
3677 | 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
20 * | |
21 */ | |
22 | |
23 /** | |
24 * @file vmnc.c | |
25 * VMware Screen Codec (VMnc) decoder | |
26 * As Alex Beregszaszi discovered, this is effectively RFB data dump | |
27 */ | |
28 | |
29 #include <stdio.h> | |
30 #include <stdlib.h> | |
31 | |
32 #include "common.h" | |
33 #include "avcodec.h" | |
34 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
35 enum EncTypes { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
36 MAGIC_WMVd = 0x574D5664, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
37 MAGIC_WMVe, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
38 MAGIC_WMVf, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
39 MAGIC_WMVg, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
40 MAGIC_WMVh, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
41 MAGIC_WMVi, |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
42 MAGIC_WMVj |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
43 }; |
3677 | 44 |
45 enum HexTile_Flags { | |
46 HT_RAW = 1, // tile is raw | |
47 HT_BKG = 2, // background color is present | |
48 HT_FG = 4, // foreground color is present | |
49 HT_SUB = 8, // subrects are present | |
50 HT_CLR = 16 // each subrect has own color | |
51 }; | |
52 | |
53 /* | |
54 * Decoder context | |
55 */ | |
56 typedef struct VmncContext { | |
57 AVCodecContext *avctx; | |
58 AVFrame pic; | |
59 | |
60 int bpp; | |
61 int bpp2; | |
62 int bigendian; | |
63 uint8_t pal[768]; | |
64 int width, height; | |
3684 | 65 |
66 /* cursor data */ | |
67 int cur_w, cur_h; | |
68 int cur_x, cur_y; | |
69 int cur_hx, cur_hy; | |
70 uint8_t* curbits, *curmask; | |
71 uint8_t* screendta; | |
3677 | 72 } VmncContext; |
73 | |
74 /* read pixel value from stream */ | |
4283
d6f83e2f8804
rename always_inline to av_always_inline and move to common.h
mru
parents:
3947
diff
changeset
|
75 static av_always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) { |
3677 | 76 switch(bpp * 2 + be) { |
77 case 2: | |
78 case 3: return *buf; | |
4364 | 79 case 4: return AV_RL16(buf); |
80 case 5: return AV_RB16(buf); | |
81 case 8: return AV_RL32(buf); | |
82 case 9: return AV_RB32(buf); | |
3677 | 83 default: return 0; |
84 } | |
85 } | |
86 | |
3684 | 87 static void load_cursor(VmncContext *c, uint8_t *src) |
88 { | |
89 int i, j, p; | |
90 const int bpp = c->bpp2; | |
91 uint8_t *dst8 = c->curbits; | |
92 uint16_t *dst16 = (uint16_t*)c->curbits; | |
93 uint32_t *dst32 = (uint32_t*)c->curbits; | |
94 | |
95 for(j = 0; j < c->cur_h; j++) { | |
96 for(i = 0; i < c->cur_w; i++) { | |
97 p = vmnc_get_pixel(src, bpp, c->bigendian); | |
98 src += bpp; | |
99 if(bpp == 1) *dst8++ = p; | |
100 if(bpp == 2) *dst16++ = p; | |
101 if(bpp == 4) *dst32++ = p; | |
102 } | |
103 } | |
104 dst8 = c->curmask; | |
105 dst16 = (uint16_t*)c->curmask; | |
106 dst32 = (uint32_t*)c->curmask; | |
107 for(j = 0; j < c->cur_h; j++) { | |
108 for(i = 0; i < c->cur_w; i++) { | |
109 p = vmnc_get_pixel(src, bpp, c->bigendian); | |
110 src += bpp; | |
111 if(bpp == 1) *dst8++ = p; | |
112 if(bpp == 2) *dst16++ = p; | |
113 if(bpp == 4) *dst32++ = p; | |
114 } | |
115 } | |
116 } | |
117 | |
118 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) | |
119 { | |
3920
3a151ccc6ed7
Remove unused variables and the corresponding warnings along with them.
diego
parents:
3800
diff
changeset
|
120 int i, j; |
3684 | 121 int w, h, x, y; |
122 w = c->cur_w; | |
123 if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x; | |
124 h = c->cur_h; | |
125 if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y; | |
126 x = c->cur_x; | |
127 y = c->cur_y; | |
128 if(x < 0) { | |
129 w += x; | |
130 x = 0; | |
131 } | |
132 if(y < 0) { | |
133 h += y; | |
134 y = 0; | |
135 } | |
136 | |
137 if((w < 1) || (h < 1)) return; | |
138 dst += x * c->bpp2 + y * stride; | |
139 | |
140 if(c->bpp2 == 1) { | |
141 uint8_t* cd = c->curbits, *msk = c->curmask; | |
142 for(j = 0; j < h; j++) { | |
143 for(i = 0; i < w; i++) | |
144 dst[i] = (dst[i] & cd[i]) ^ msk[i]; | |
145 msk += c->cur_w; | |
146 cd += c->cur_w; | |
147 dst += stride; | |
148 } | |
149 } else if(c->bpp2 == 2) { | |
150 uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; | |
151 uint16_t* dst2; | |
152 for(j = 0; j < h; j++) { | |
153 dst2 = (uint16_t*)dst; | |
154 for(i = 0; i < w; i++) | |
155 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
156 msk += c->cur_w; | |
157 cd += c->cur_w; | |
158 dst += stride; | |
159 } | |
160 } else if(c->bpp2 == 4) { | |
161 uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; | |
162 uint32_t* dst2; | |
163 for(j = 0; j < h; j++) { | |
164 dst2 = (uint32_t*)dst; | |
165 for(i = 0; i < w; i++) | |
166 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
167 msk += c->cur_w; | |
168 cd += c->cur_w; | |
169 dst += stride; | |
170 } | |
171 } | |
172 } | |
173 | |
3677 | 174 /* fill rectangle with given colour */ |
4283
d6f83e2f8804
rename always_inline to av_always_inline and move to common.h
mru
parents:
3947
diff
changeset
|
175 static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride) |
3677 | 176 { |
177 int i, j; | |
178 dst += dx * bpp + dy * stride; | |
179 if(bpp == 1){ | |
180 for(j = 0; j < h; j++) { | |
181 memset(dst, color, w); | |
182 dst += stride; | |
183 } | |
184 }else if(bpp == 2){ | |
185 uint16_t* dst2; | |
186 for(j = 0; j < h; j++) { | |
187 dst2 = (uint16_t*)dst; | |
188 for(i = 0; i < w; i++) { | |
189 *dst2++ = color; | |
190 } | |
191 dst += stride; | |
192 } | |
193 }else if(bpp == 4){ | |
194 uint32_t* dst2; | |
195 for(j = 0; j < h; j++) { | |
196 dst2 = (uint32_t*)dst; | |
197 for(i = 0; i < w; i++) { | |
198 dst2[i] = color; | |
199 } | |
200 dst += stride; | |
201 } | |
202 } | |
203 } | |
204 | |
4283
d6f83e2f8804
rename always_inline to av_always_inline and move to common.h
mru
parents:
3947
diff
changeset
|
205 static av_always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride) |
3677 | 206 { |
207 int i, j, p; | |
208 for(j = 0; j < h; j++) { | |
209 for(i = 0; i < w; i++) { | |
210 p = vmnc_get_pixel(src, bpp, be); | |
211 src += bpp; | |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
212 switch(bpp){ |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
213 case 1: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
214 dst[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
215 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
216 case 2: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
217 ((uint16_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
218 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
219 case 4: |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
220 ((uint32_t*)dst)[i] = p; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
221 break; |
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
222 } |
3677 | 223 } |
224 dst += stride; | |
225 } | |
226 } | |
227 | |
3685 | 228 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int ssize, int w, int h, int stride) |
3677 | 229 { |
230 int i, j, k; | |
231 int bg = 0, fg = 0, rects, color, flags, xy, wh; | |
232 const int bpp = c->bpp2; | |
233 uint8_t *dst2; | |
234 int bw = 16, bh = 16; | |
235 uint8_t *ssrc=src; | |
236 | |
237 for(j = 0; j < h; j += 16) { | |
238 dst2 = dst; | |
239 bw = 16; | |
240 if(j + 16 > h) bh = h - j; | |
241 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) { | |
3685 | 242 if(src - ssrc >= ssize) { |
243 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
244 return -1; | |
245 } | |
3677 | 246 if(i + 16 > w) bw = w - i; |
247 flags = *src++; | |
248 if(flags & HT_RAW) { | |
3685 | 249 if(src - ssrc > ssize - bw * bh * bpp) { |
250 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
251 return -1; | |
252 } | |
3677 | 253 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride); |
3678
93a961e40d65
Handle raw blocks correctly (both updating pointer and storing to memory)
kostya
parents:
3677
diff
changeset
|
254 src += bw * bh * bpp; |
3677 | 255 } else { |
256 if(flags & HT_BKG) { | |
257 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
258 } | |
259 if(flags & HT_FG) { | |
260 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
261 } | |
262 rects = 0; | |
263 if(flags & HT_SUB) | |
264 rects = *src++; | |
3685 | 265 color = !!(flags & HT_CLR); |
3677 | 266 |
267 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); | |
268 | |
3685 | 269 if(src - ssrc > ssize - rects * (color * bpp + 2)) { |
270 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
271 return -1; | |
272 } | |
3677 | 273 for(k = 0; k < rects; k++) { |
274 if(color) { | |
275 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; | |
276 } | |
277 xy = *src++; | |
278 wh = *src++; | |
279 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); | |
280 } | |
281 } | |
282 } | |
283 dst += stride * 16; | |
284 } | |
285 return src - ssrc; | |
286 } | |
287 | |
288 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size) | |
289 { | |
290 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
291 uint8_t *outptr; | |
292 uint8_t *src = buf; | |
3685 | 293 int dx, dy, w, h, depth, enc, chunks, res, size_left; |
3677 | 294 |
295 c->pic.reference = 1; | |
296 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; | |
297 if(avctx->reget_buffer(avctx, &c->pic) < 0){ | |
298 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); | |
299 return -1; | |
300 } | |
301 | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
302 c->pic.key_frame = 0; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
303 c->pic.pict_type = FF_P_TYPE; |
3677 | 304 |
3684 | 305 //restore screen after cursor |
306 if(c->screendta) { | |
307 int i; | |
308 w = c->cur_w; | |
309 if(c->width < c->cur_x + w) w = c->width - c->cur_x; | |
310 h = c->cur_h; | |
311 if(c->height < c->cur_y + h) h = c->height - c->cur_y; | |
312 dx = c->cur_x; | |
313 if(dx < 0) { | |
314 w += dx; | |
315 dx = 0; | |
316 } | |
317 dy = c->cur_y; | |
318 if(dy < 0) { | |
319 h += dy; | |
320 dy = 0; | |
321 } | |
322 if((w > 0) && (h > 0)) { | |
323 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; | |
324 for(i = 0; i < h; i++) { | |
325 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2); | |
326 outptr += c->pic.linesize[0]; | |
327 } | |
328 } | |
329 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
330 src += 2; |
4364 | 331 chunks = AV_RB16(src); src += 2; |
3677 | 332 while(chunks--) { |
4364 | 333 dx = AV_RB16(src); src += 2; |
334 dy = AV_RB16(src); src += 2; | |
335 w = AV_RB16(src); src += 2; | |
336 h = AV_RB16(src); src += 2; | |
337 enc = AV_RB32(src); src += 4; | |
3684 | 338 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; |
3685 | 339 size_left = buf_size - (src - buf); |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
340 switch(enc) { |
3684 | 341 case MAGIC_WMVd: // cursor |
3685 | 342 if(size_left < 2 + w * h * c->bpp2 * 2) { |
343 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left); | |
344 return -1; | |
345 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
346 src += 2; |
3684 | 347 c->cur_w = w; |
348 c->cur_h = h; | |
349 c->cur_hx = dx; | |
350 c->cur_hy = dy; | |
351 if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { | |
352 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); | |
353 c->cur_hx = c->cur_hy = 0; | |
354 } | |
355 c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2); | |
356 c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2); | |
357 c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2); | |
358 load_cursor(c, src); | |
359 src += w * h * c->bpp2 * 2; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
360 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
361 case MAGIC_WMVe: // unknown |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
362 src += 2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
363 break; |
3684 | 364 case MAGIC_WMVf: // update cursor position |
365 c->cur_x = dx - c->cur_hx; | |
366 c->cur_y = dy - c->cur_hy; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
367 break; |
3686 | 368 case MAGIC_WMVg: // unknown |
369 src += 10; | |
370 break; | |
371 case MAGIC_WMVh: // unknown | |
372 src += 4; | |
373 break; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
374 case MAGIC_WMVi: // ServerInitialization struct |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
375 c->pic.key_frame = 1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
376 c->pic.pict_type = FF_I_TYPE; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
377 depth = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
378 if(depth != c->bpp) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
379 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
380 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
381 src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
382 c->bigendian = *src++; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
383 if(c->bigendian & (~1)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
384 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
385 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
386 } |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
387 //skip the rest of pixel format data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
388 src += 13; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
389 break; |
3686 | 390 case MAGIC_WMVj: // unknown |
391 src += 2; | |
392 break; | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
393 case 0x00000000: // raw rectangle data |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
394 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
395 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
396 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
397 } |
3685 | 398 if(size_left < w * h * c->bpp2) { |
399 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left); | |
400 return -1; | |
401 } | |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
402 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
403 src += w * h * c->bpp2; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
404 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
405 case 0x00000005: // HexTile encoded rectangle |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
406 if((dx + w > c->width) || (dy + h > c->height)) { |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
407 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
408 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
409 } |
3685 | 410 res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]); |
3679
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
411 if(res < 0) |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
412 return -1; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
413 src += res; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
414 break; |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
415 default: |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
416 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); |
2702eec8c8b9
Try to handle all chunks, previous scheme was not correct.
kostya
parents:
3678
diff
changeset
|
417 chunks = 0; // leave chunks decoding loop |
3677 | 418 } |
419 } | |
3684 | 420 if(c->screendta){ |
421 int i; | |
422 //save screen data before painting cursor | |
423 w = c->cur_w; | |
424 if(c->width < c->cur_x + w) w = c->width - c->cur_x; | |
425 h = c->cur_h; | |
426 if(c->height < c->cur_y + h) h = c->height - c->cur_y; | |
427 dx = c->cur_x; | |
428 if(dx < 0) { | |
429 w += dx; | |
430 dx = 0; | |
431 } | |
432 dy = c->cur_y; | |
433 if(dy < 0) { | |
434 h += dy; | |
435 dy = 0; | |
436 } | |
437 if((w > 0) && (h > 0)) { | |
438 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; | |
439 for(i = 0; i < h; i++) { | |
440 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2); | |
441 outptr += c->pic.linesize[0]; | |
442 } | |
443 outptr = c->pic.data[0]; | |
444 put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y); | |
445 } | |
446 } | |
3677 | 447 *data_size = sizeof(AVFrame); |
448 *(AVFrame*)data = c->pic; | |
449 | |
450 /* always report that the buffer was completely consumed */ | |
451 return buf_size; | |
452 } | |
453 | |
454 | |
455 | |
456 /* | |
457 * | |
458 * Init VMnc decoder | |
459 * | |
460 */ | |
461 static int decode_init(AVCodecContext *avctx) | |
462 { | |
463 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
464 | |
465 c->avctx = avctx; | |
466 avctx->has_b_frames = 0; | |
467 | |
468 c->pic.data[0] = NULL; | |
469 c->width = avctx->width; | |
470 c->height = avctx->height; | |
471 | |
3800
9b75ab171fa9
1l: correct argument order in avcodec_check_dimensions
kostya
parents:
3686
diff
changeset
|
472 if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) { |
3677 | 473 return 1; |
474 } | |
475 c->bpp = avctx->bits_per_sample; | |
476 c->bpp2 = c->bpp/8; | |
477 | |
478 switch(c->bpp){ | |
479 case 8: | |
480 avctx->pix_fmt = PIX_FMT_PAL8; | |
481 break; | |
482 case 16: | |
483 avctx->pix_fmt = PIX_FMT_RGB555; | |
484 break; | |
485 case 32: | |
486 avctx->pix_fmt = PIX_FMT_RGB32; | |
487 break; | |
488 default: | |
489 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); | |
490 } | |
491 | |
492 return 0; | |
493 } | |
494 | |
495 | |
496 | |
497 /* | |
498 * | |
499 * Uninit VMnc decoder | |
500 * | |
501 */ | |
502 static int decode_end(AVCodecContext *avctx) | |
503 { | |
504 VmncContext * const c = (VmncContext *)avctx->priv_data; | |
505 | |
506 if (c->pic.data[0]) | |
507 avctx->release_buffer(avctx, &c->pic); | |
508 | |
3684 | 509 av_free(c->curbits); |
510 av_free(c->curmask); | |
511 av_free(c->screendta); | |
3677 | 512 return 0; |
513 } | |
514 | |
515 AVCodec vmnc_decoder = { | |
516 "VMware video", | |
517 CODEC_TYPE_VIDEO, | |
518 CODEC_ID_VMNC, | |
519 sizeof(VmncContext), | |
520 decode_init, | |
521 NULL, | |
522 decode_end, | |
523 decode_frame | |
524 }; | |
525 |