Mercurial > libavcodec.hg

comparison vmdav.c @ 2829:9ff4c2733422 libavcodec

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tinfoil patch: be more diligent about checking array boundaries before writing to them
author melanson
date Sat, 13 Aug 2005 18:39:21 +0000
parents 4bdd4927d2fc
children ef2149182f1c
comparison
equal deleted inserted replaced
2828:2aae25679885 2829:9ff4c2733422
64 unsigned char *buf; 64 unsigned char *buf;
65 int size; 65 int size;
66 66
67 unsigned char palette[PALETTE_COUNT * 4]; 67 unsigned char palette[PALETTE_COUNT * 4];
68 unsigned char *unpack_buffer; 68 unsigned char *unpack_buffer;
69 int unpack_buffer_size;
69 70
70 } VmdVideoContext; 71 } VmdVideoContext;
71 72
72 #define QUEUE_SIZE 0x1000 73 #define QUEUE_SIZE 0x1000
73 #define QUEUE_MASK 0x0FFF 74 #define QUEUE_MASK 0x0FFF
74 75
75 static void lz_unpack(unsigned char *src, unsigned char *dest) 76 static void lz_unpack(unsigned char *src, unsigned char *dest, int dest_len)
76 { 77 {
77 unsigned char *s; 78 unsigned char *s;
78 unsigned char *d; 79 unsigned char *d;
80 unsigned char *d_end;
79 unsigned char queue[QUEUE_SIZE]; 81 unsigned char queue[QUEUE_SIZE];
80 unsigned int qpos; 82 unsigned int qpos;
81 unsigned int dataleft; 83 unsigned int dataleft;
82 unsigned int chainofs; 84 unsigned int chainofs;
83 unsigned int chainlen; 85 unsigned int chainlen;
85 unsigned char tag; 87 unsigned char tag;
86 unsigned int i, j; 88 unsigned int i, j;
87 89
88 s = src; 90 s = src;
89 d = dest; 91 d = dest;
92 d_end = d + dest_len;
90 dataleft = LE_32(s); 93 dataleft = LE_32(s);
91 s += 4; 94 s += 4;
92 memset(queue, QUEUE_SIZE, 0x20); 95 memset(queue, QUEUE_SIZE, 0x20);
93 if (LE_32(s) == 0x56781234) { 96 if (LE_32(s) == 0x56781234) {
94 s += 4; 97 s += 4;
100 } 103 }
101 104
102 while (dataleft > 0) { 105 while (dataleft > 0) {
103 tag = *s++; 106 tag = *s++;
104 if ((tag == 0xFF) && (dataleft > 8)) { 107 if ((tag == 0xFF) && (dataleft > 8)) {
108 if (d + 8 > d_end)
109 return;
105 for (i = 0; i < 8; i++) { 110 for (i = 0; i < 8; i++) {
106 queue[qpos++] = *d++ = *s++; 111 queue[qpos++] = *d++ = *s++;
107 qpos &= QUEUE_MASK; 112 qpos &= QUEUE_MASK;
108 } 113 }
109 dataleft -= 8; 114 dataleft -= 8;
110 } else { 115 } else {
111 for (i = 0; i < 8; i++) { 116 for (i = 0; i < 8; i++) {
112 if (dataleft == 0) 117 if (dataleft == 0)
113 break; 118 break;
114 if (tag & 0x01) { 119 if (tag & 0x01) {
120 if (d + 1 > d_end)
121 return;
115 queue[qpos++] = *d++ = *s++; 122 queue[qpos++] = *d++ = *s++;
116 qpos &= QUEUE_MASK; 123 qpos &= QUEUE_MASK;
117 dataleft--; 124 dataleft--;
118 } else { 125 } else {
119 chainofs = *s++; 126 chainofs = *s++;
120 chainofs |= ((*s & 0xF0) << 4); 127 chainofs |= ((*s & 0xF0) << 4);
121 chainlen = (*s++ & 0x0F) + 3; 128 chainlen = (*s++ & 0x0F) + 3;
122 if (chainlen == speclen) 129 if (chainlen == speclen)
123 chainlen = *s++ + 0xF + 3; 130 chainlen = *s++ + 0xF + 3;
131 if (d + chainlen > d_end)
132 return;
124 for (j = 0; j < chainlen; j++) { 133 for (j = 0; j < chainlen; j++) {
125 *d = queue[chainofs++ & QUEUE_MASK]; 134 *d = queue[chainofs++ & QUEUE_MASK];
126 queue[qpos++] = *d++; 135 queue[qpos++] = *d++;
127 qpos &= QUEUE_MASK; 136 qpos &= QUEUE_MASK;
128 } 137 }
132 } 141 }
133 } 142 }
134 } 143 }
135 } 144 }
136 145
137 static int rle_unpack(unsigned char *src, unsigned char *dest, int len) 146 static int rle_unpack(unsigned char *src, unsigned char *dest,
147 int src_len, int dest_len)
138 { 148 {
139 unsigned char *ps; 149 unsigned char *ps;
140 unsigned char *pd; 150 unsigned char *pd;
141 int i, l; 151 int i, l;
152 unsigned char *dest_end = dest + dest_len;
142 153
143 ps = src; 154 ps = src;
144 pd = dest; 155 pd = dest;
145 if (len & 1) 156 if (src_len & 1)
146 *pd++ = *ps++; 157 *pd++ = *ps++;
147 158
148 len >>= 1; 159 src_len >>= 1;
149 i = 0; 160 i = 0;
150 do { 161 do {
151 l = *ps++; 162 l = *ps++;
152 if (l & 0x80) { 163 if (l & 0x80) {
153 l = (l & 0x7F) * 2; 164 l = (l & 0x7F) * 2;
165 if (pd + l > dest_end)
166 return (ps - src);
154 memcpy(pd, ps, l); 167 memcpy(pd, ps, l);
155 ps += l; 168 ps += l;
156 pd += l; 169 pd += l;
157 } else { 170 } else {
171 if (pd + i > dest_end)
172 return (ps - src);
158 for (i = 0; i < l; i++) { 173 for (i = 0; i < l; i++) {
159 *pd++ = ps[0]; 174 *pd++ = ps[0];
160 *pd++ = ps[1]; 175 *pd++ = ps[1];
161 } 176 }
162 ps += 2; 177 ps += 2;
163 } 178 }
164 i += l; 179 i += l;
165 } while (i < len); 180 } while (i < src_len);
166 181
167 return (ps - src); 182 return (ps - src);
168 } 183 }
169 184
170 static void vmd_decode(VmdVideoContext *s) 185 static void vmd_decode(VmdVideoContext *s)
183 unsigned char len; 198 unsigned char len;
184 int ofs; 199 int ofs;
185 200
186 int frame_x, frame_y; 201 int frame_x, frame_y;
187 int frame_width, frame_height; 202 int frame_width, frame_height;
203 int dp_size;
188 204
189 frame_x = LE_16(&s->buf[6]); 205 frame_x = LE_16(&s->buf[6]);
190 frame_y = LE_16(&s->buf[8]); 206 frame_y = LE_16(&s->buf[8]);
191 frame_width = LE_16(&s->buf[10]) - frame_x + 1; 207 frame_width = LE_16(&s->buf[10]) - frame_x + 1;
192 frame_height = LE_16(&s->buf[12]) - frame_y + 1; 208 frame_height = LE_16(&s->buf[12]) - frame_y + 1;
215 if (s->size >= 0) { 231 if (s->size >= 0) {
216 /* originally UnpackFrame in VAG's code */ 232 /* originally UnpackFrame in VAG's code */
217 pb = p; 233 pb = p;
218 meth = *pb++; 234 meth = *pb++;
219 if (meth & 0x80) { 235 if (meth & 0x80) {
220 lz_unpack(pb, s->unpack_buffer); 236 lz_unpack(pb, s->unpack_buffer, s->unpack_buffer_size);
221 meth &= 0x7F; 237 meth &= 0x7F;
222 pb = s->unpack_buffer; 238 pb = s->unpack_buffer;
223 } 239 }
224 240
225 dp = &s->frame.data[0][frame_y * s->frame.linesize[0] + frame_x]; 241 dp = &s->frame.data[0][frame_y * s->frame.linesize[0] + frame_x];
242 dp_size = s->frame.linesize[0] * s->avctx->height;
226 pp = &s->prev_frame.data[0][frame_y * s->prev_frame.linesize[0] + frame_x]; 243 pp = &s->prev_frame.data[0][frame_y * s->prev_frame.linesize[0] + frame_x];
227 switch (meth) { 244 switch (meth) {
228 case 1: 245 case 1:
229 for (i = 0; i < frame_height; i++) { 246 for (i = 0; i < frame_height; i++) {
230 ofs = 0; 247 ofs = 0;
231 do { 248 do {
232 len = *pb++; 249 len = *pb++;
233 if (len & 0x80) { 250 if (len & 0x80) {
234 len = (len & 0x7F) + 1; 251 len = (len & 0x7F) + 1;
252 if (ofs + len > frame_width)
253 return;
235 memcpy(&dp[ofs], pb, len); 254 memcpy(&dp[ofs], pb, len);
236 pb += len; 255 pb += len;
237 ofs += len; 256 ofs += len;
238 } else { 257 } else {
239 /* interframe pixel copy */ 258 /* interframe pixel copy */
259 if (ofs + len + 1 > frame_width)
260 return;
240 memcpy(&dp[ofs], &pp[ofs], len + 1); 261 memcpy(&dp[ofs], &pp[ofs], len + 1);
241 ofs += len + 1; 262 ofs += len + 1;
242 } 263 }
243 } while (ofs < frame_width); 264 } while (ofs < frame_width);
244 if (ofs > frame_width) { 265 if (ofs > frame_width) {
266 do { 287 do {
267 len = *pb++; 288 len = *pb++;
268 if (len & 0x80) { 289 if (len & 0x80) {
269 len = (len & 0x7F) + 1; 290 len = (len & 0x7F) + 1;
270 if (*pb++ == 0xFF) 291 if (*pb++ == 0xFF)
271 len = rle_unpack(pb, &dp[ofs], len); 292 len = rle_unpack(pb, &dp[ofs], len, frame_width - ofs);
272 else 293 else
273 memcpy(&dp[ofs], pb, len); 294 memcpy(&dp[ofs], pb, len);
274 pb += len; 295 pb += len;
275 ofs += len; 296 ofs += len;
276 } else { 297 } else {
277 /* interframe pixel copy */ 298 /* interframe pixel copy */
299 if (ofs + len + 1 > frame_width)
300 return;
278 memcpy(&dp[ofs], &pp[ofs], len + 1); 301 memcpy(&dp[ofs], &pp[ofs], len + 1);
279 ofs += len + 1; 302 ofs += len + 1;
280 } 303 }
281 } while (ofs < frame_width); 304 } while (ofs < frame_width);
282 if (ofs > frame_width) { 305 if (ofs > frame_width) {
312 VMD_HEADER_SIZE); 335 VMD_HEADER_SIZE);
313 return -1; 336 return -1;
314 } 337 }
315 vmd_header = (unsigned char *)avctx->extradata; 338 vmd_header = (unsigned char *)avctx->extradata;
316 339
317 s->unpack_buffer = av_malloc(LE_32(&vmd_header[800])); 340 s->unpack_buffer_size = LE_32(&vmd_header[800]);
341 s->unpack_buffer = av_malloc(s->unpack_buffer_size);
318 if (!s->unpack_buffer) 342 if (!s->unpack_buffer)
319 return -1; 343 return -1;
320 344
321 /* load up the initial palette */ 345 /* load up the initial palette */
322 raw_palette = &vmd_header[28]; 346 raw_palette = &vmd_header[28];