Mercurial > libavcodec.hg
annotate pnmenc.c @ 6018:9d1654835629 libavcodec
Ensure that our total reference frame count does not exceed the SPS
max frame count, which is limited to less than the size of the
reference buffers, thereby preventing overflow.
Part of fix for issue 281.
author | heydowns |
---|---|
date | Fri, 14 Dec 2007 06:25:23 +0000 |
parents | bff60ecc02f9 |
children | ec8c482d17c1 |
rev | line source |
---|---|
2344 | 1 /* |
2 * PNM image format | |
3 * Copyright (c) 2002, 2003 Fabrice Bellard. | |
4 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
5 * This file is part of FFmpeg. |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
6 * |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
7 * FFmpeg is free software; you can redistribute it and/or |
2344 | 8 * modify it under the terms of the GNU Lesser General Public |
9 * License as published by the Free Software Foundation; either | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
10 * version 2.1 of the License, or (at your option) any later version. |
2344 | 11 * |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
12 * FFmpeg is distributed in the hope that it will be useful, |
2344 | 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 * Lesser General Public License for more details. | |
16 * | |
17 * You should have received a copy of the GNU Lesser General Public | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3455
diff
changeset
|
18 * License along with FFmpeg; if not, write to the Free Software |
3036
0b546eab515d
Update licensing information: The FSF changed postal address.
diego
parents:
2967
diff
changeset
|
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
2344 | 20 */ |
21 #include "avcodec.h" | |
5089 | 22 #include "bytestream.h" |
4978 | 23 #include "pnm.h" |
2967 | 24 |
2344 | 25 |
26 static int common_init(AVCodecContext *avctx){ | |
27 PNMContext *s = avctx->priv_data; | |
28 | |
29 avcodec_get_frame_defaults((AVFrame*)&s->picture); | |
30 avctx->coded_frame= (AVFrame*)&s->picture; | |
31 | |
32 return 0; | |
33 } | |
34 | |
2967 | 35 static int pnm_decode_frame(AVCodecContext *avctx, |
2348 | 36 void *data, int *data_size, |
37 uint8_t *buf, int buf_size) | |
38 { | |
39 PNMContext * const s = avctx->priv_data; | |
40 AVFrame *picture = data; | |
41 AVFrame * const p= (AVFrame*)&s->picture; | |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
42 int i, n, linesize, h, upgrade = 0; |
2348 | 43 unsigned char *ptr; |
44 | |
45 s->bytestream_start= | |
46 s->bytestream= buf; | |
47 s->bytestream_end= buf + buf_size; | |
2967 | 48 |
4978 | 49 if(ff_pnm_decode_header(avctx, s) < 0) |
2349 | 50 return -1; |
2967 | 51 |
2344 | 52 if(p->data[0]) |
53 avctx->release_buffer(avctx, p); | |
54 | |
55 p->reference= 0; | |
56 if(avctx->get_buffer(avctx, p) < 0){ | |
57 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); | |
58 return -1; | |
59 } | |
60 p->pict_type= FF_I_TYPE; | |
61 p->key_frame= 1; | |
2967 | 62 |
2344 | 63 switch(avctx->pix_fmt) { |
64 default: | |
65 return -1; | |
66 case PIX_FMT_RGB24: | |
67 n = avctx->width * 3; | |
68 goto do_read; | |
69 case PIX_FMT_GRAY8: | |
70 n = avctx->width; | |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
71 if (s->maxval < 255) |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
72 upgrade = 1; |
2344 | 73 goto do_read; |
4068 | 74 case PIX_FMT_GRAY16BE: |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
75 case PIX_FMT_GRAY16LE: |
4068 | 76 n = avctx->width * 2; |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
77 if (s->maxval < 65535) |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
78 upgrade = 2; |
4068 | 79 goto do_read; |
2344 | 80 case PIX_FMT_MONOWHITE: |
2349 | 81 case PIX_FMT_MONOBLACK: |
2344 | 82 n = (avctx->width + 7) >> 3; |
83 do_read: | |
84 ptr = p->data[0]; | |
85 linesize = p->linesize[0]; | |
2839 | 86 if(s->bytestream + n*avctx->height > s->bytestream_end) |
87 return -1; | |
2344 | 88 for(i = 0; i < avctx->height; i++) { |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
89 if (!upgrade) |
4837 | 90 memcpy(ptr, s->bytestream, n); |
4836
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
91 else if (upgrade == 1) { |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
92 unsigned int j, f = (255*128 + s->maxval/2) / s->maxval; |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
93 for (j=0; j<n; j++) |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
94 ptr[j] = (s->bytestream[j] * f + 64) >> 7; |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
95 } else if (upgrade == 2) { |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
96 unsigned int j, v, f = (65535*32768 + s->maxval/2) / s->maxval; |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
97 for (j=0; j<n/2; j++) { |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
98 v = be2me_16(((uint16_t *)s->bytestream)[j]); |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
99 ((uint16_t *)ptr)[j] = (v * f + 16384) >> 15; |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
100 } |
ae19d863073f
Add support for grayscale images with arbitrary maxvals.
ivo
parents:
4494
diff
changeset
|
101 } |
2344 | 102 s->bytestream += n; |
103 ptr += linesize; | |
104 } | |
105 break; | |
106 case PIX_FMT_YUV420P: | |
107 { | |
108 unsigned char *ptr1, *ptr2; | |
109 | |
110 n = avctx->width; | |
111 ptr = p->data[0]; | |
112 linesize = p->linesize[0]; | |
2839 | 113 if(s->bytestream + n*avctx->height*3/2 > s->bytestream_end) |
114 return -1; | |
2344 | 115 for(i = 0; i < avctx->height; i++) { |
116 memcpy(ptr, s->bytestream, n); | |
117 s->bytestream += n; | |
118 ptr += linesize; | |
119 } | |
120 ptr1 = p->data[1]; | |
121 ptr2 = p->data[2]; | |
122 n >>= 1; | |
123 h = avctx->height >> 1; | |
124 for(i = 0; i < h; i++) { | |
125 memcpy(ptr1, s->bytestream, n); | |
126 s->bytestream += n; | |
127 memcpy(ptr2, s->bytestream, n); | |
128 s->bytestream += n; | |
129 ptr1 += p->linesize[1]; | |
130 ptr2 += p->linesize[2]; | |
131 } | |
132 } | |
133 break; | |
4494
ce643a22f049
Replace deprecated PIX_FMT names by the newer variants.
diego
parents:
4152
diff
changeset
|
134 case PIX_FMT_RGB32: |
2349 | 135 ptr = p->data[0]; |
136 linesize = p->linesize[0]; | |
2839 | 137 if(s->bytestream + avctx->width*avctx->height*4 > s->bytestream_end) |
138 return -1; | |
2349 | 139 for(i = 0; i < avctx->height; i++) { |
140 int j, r, g, b, a; | |
141 | |
142 for(j = 0;j < avctx->width; j++) { | |
143 r = *s->bytestream++; | |
144 g = *s->bytestream++; | |
145 b = *s->bytestream++; | |
146 a = *s->bytestream++; | |
147 ((uint32_t *)ptr)[j] = (a << 24) | (r << 16) | (g << 8) | b; | |
148 } | |
149 ptr += linesize; | |
150 } | |
151 break; | |
2344 | 152 } |
153 *picture= *(AVFrame*)&s->picture; | |
154 *data_size = sizeof(AVPicture); | |
155 | |
156 return s->bytestream - s->bytestream_start; | |
157 } | |
158 | |
159 static int pnm_encode_frame(AVCodecContext *avctx, unsigned char *outbuf, int buf_size, void *data){ | |
160 PNMContext *s = avctx->priv_data; | |
161 AVFrame *pict = data; | |
162 AVFrame * const p= (AVFrame*)&s->picture; | |
163 int i, h, h1, c, n, linesize; | |
164 uint8_t *ptr, *ptr1, *ptr2; | |
165 | |
2422 | 166 if(buf_size < avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height) + 200){ |
167 av_log(avctx, AV_LOG_ERROR, "encoded frame too large\n"); | |
168 return -1; | |
169 } | |
170 | |
2344 | 171 *p = *pict; |
172 p->pict_type= FF_I_TYPE; | |
173 p->key_frame= 1; | |
2967 | 174 |
2344 | 175 s->bytestream_start= |
176 s->bytestream= outbuf; | |
177 s->bytestream_end= outbuf+buf_size; | |
178 | |
179 h = avctx->height; | |
180 h1 = h; | |
181 switch(avctx->pix_fmt) { | |
182 case PIX_FMT_MONOWHITE: | |
183 c = '4'; | |
184 n = (avctx->width + 7) >> 3; | |
185 break; | |
186 case PIX_FMT_GRAY8: | |
187 c = '5'; | |
188 n = avctx->width; | |
189 break; | |
4068 | 190 case PIX_FMT_GRAY16BE: |
191 c = '5'; | |
192 n = avctx->width * 2; | |
193 break; | |
2344 | 194 case PIX_FMT_RGB24: |
195 c = '6'; | |
196 n = avctx->width * 3; | |
197 break; | |
198 case PIX_FMT_YUV420P: | |
199 c = '5'; | |
200 n = avctx->width; | |
201 h1 = (h * 3) / 2; | |
202 break; | |
203 default: | |
204 return -1; | |
205 } | |
2967 | 206 snprintf(s->bytestream, s->bytestream_end - s->bytestream, |
2344 | 207 "P%c\n%d %d\n", |
208 c, avctx->width, h1); | |
209 s->bytestream += strlen(s->bytestream); | |
210 if (avctx->pix_fmt != PIX_FMT_MONOWHITE) { | |
2967 | 211 snprintf(s->bytestream, s->bytestream_end - s->bytestream, |
4068 | 212 "%d\n", (avctx->pix_fmt != PIX_FMT_GRAY16BE) ? 255 : 65535); |
2344 | 213 s->bytestream += strlen(s->bytestream); |
214 } | |
215 | |
216 ptr = p->data[0]; | |
217 linesize = p->linesize[0]; | |
218 for(i=0;i<h;i++) { | |
219 memcpy(s->bytestream, ptr, n); | |
220 s->bytestream += n; | |
221 ptr += linesize; | |
222 } | |
2967 | 223 |
2344 | 224 if (avctx->pix_fmt == PIX_FMT_YUV420P) { |
225 h >>= 1; | |
226 n >>= 1; | |
227 ptr1 = p->data[1]; | |
228 ptr2 = p->data[2]; | |
229 for(i=0;i<h;i++) { | |
230 memcpy(s->bytestream, ptr1, n); | |
231 s->bytestream += n; | |
232 memcpy(s->bytestream, ptr2, n); | |
233 s->bytestream += n; | |
234 ptr1 += p->linesize[1]; | |
235 ptr2 += p->linesize[2]; | |
236 } | |
237 } | |
238 return s->bytestream - s->bytestream_start; | |
239 } | |
240 | |
241 static int pam_encode_frame(AVCodecContext *avctx, unsigned char *outbuf, int buf_size, void *data){ | |
242 PNMContext *s = avctx->priv_data; | |
243 AVFrame *pict = data; | |
244 AVFrame * const p= (AVFrame*)&s->picture; | |
245 int i, h, w, n, linesize, depth, maxval; | |
246 const char *tuple_type; | |
247 uint8_t *ptr; | |
248 | |
2422 | 249 if(buf_size < avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height) + 200){ |
250 av_log(avctx, AV_LOG_ERROR, "encoded frame too large\n"); | |
251 return -1; | |
252 } | |
253 | |
2344 | 254 *p = *pict; |
255 p->pict_type= FF_I_TYPE; | |
256 p->key_frame= 1; | |
2967 | 257 |
2344 | 258 s->bytestream_start= |
259 s->bytestream= outbuf; | |
260 s->bytestream_end= outbuf+buf_size; | |
261 | |
262 h = avctx->height; | |
263 w = avctx->width; | |
264 switch(avctx->pix_fmt) { | |
265 case PIX_FMT_MONOWHITE: | |
266 n = (w + 7) >> 3; | |
267 depth = 1; | |
268 maxval = 1; | |
269 tuple_type = "BLACKANDWHITE"; | |
270 break; | |
271 case PIX_FMT_GRAY8: | |
272 n = w; | |
273 depth = 1; | |
274 maxval = 255; | |
275 tuple_type = "GRAYSCALE"; | |
276 break; | |
277 case PIX_FMT_RGB24: | |
278 n = w * 3; | |
279 depth = 3; | |
280 maxval = 255; | |
281 tuple_type = "RGB"; | |
282 break; | |
4494
ce643a22f049
Replace deprecated PIX_FMT names by the newer variants.
diego
parents:
4152
diff
changeset
|
283 case PIX_FMT_RGB32: |
2344 | 284 n = w * 4; |
285 depth = 4; | |
286 maxval = 255; | |
287 tuple_type = "RGB_ALPHA"; | |
288 break; | |
289 default: | |
290 return -1; | |
291 } | |
2967 | 292 snprintf(s->bytestream, s->bytestream_end - s->bytestream, |
2344 | 293 "P7\nWIDTH %d\nHEIGHT %d\nDEPTH %d\nMAXVAL %d\nTUPLETYPE %s\nENDHDR\n", |
294 w, h, depth, maxval, tuple_type); | |
295 s->bytestream += strlen(s->bytestream); | |
2967 | 296 |
2344 | 297 ptr = p->data[0]; |
298 linesize = p->linesize[0]; | |
2967 | 299 |
4494
ce643a22f049
Replace deprecated PIX_FMT names by the newer variants.
diego
parents:
4152
diff
changeset
|
300 if (avctx->pix_fmt == PIX_FMT_RGB32) { |
2344 | 301 int j; |
302 unsigned int v; | |
303 | |
304 for(i=0;i<h;i++) { | |
305 for(j=0;j<w;j++) { | |
306 v = ((uint32_t *)ptr)[j]; | |
5089 | 307 bytestream_put_be24(&s->bytestream, v); |
2344 | 308 *s->bytestream++ = v >> 24; |
309 } | |
310 ptr += linesize; | |
311 } | |
312 } else { | |
313 for(i=0;i<h;i++) { | |
314 memcpy(s->bytestream, ptr, n); | |
315 s->bytestream += n; | |
316 ptr += linesize; | |
317 } | |
318 } | |
319 return s->bytestream - s->bytestream_start; | |
320 } | |
321 | |
322 #if 0 | |
323 static int pnm_probe(AVProbeData *pd) | |
324 { | |
325 const char *p = pd->buf; | |
326 if (pd->buf_size >= 8 && | |
327 p[0] == 'P' && | |
328 p[1] >= '4' && p[1] <= '6' && | |
329 pnm_space(p[2]) ) | |
330 return AVPROBE_SCORE_MAX - 1; /* to permit pgmyuv probe */ | |
331 else | |
332 return 0; | |
333 } | |
334 | |
335 static int pgmyuv_probe(AVProbeData *pd) | |
336 { | |
337 if (match_ext(pd->filename, "pgmyuv")) | |
338 return AVPROBE_SCORE_MAX; | |
339 else | |
340 return 0; | |
341 } | |
342 | |
343 static int pam_probe(AVProbeData *pd) | |
344 { | |
345 const char *p = pd->buf; | |
346 if (pd->buf_size >= 8 && | |
347 p[0] == 'P' && | |
348 p[1] == '7' && | |
349 p[2] == '\n') | |
350 return AVPROBE_SCORE_MAX; | |
351 else | |
352 return 0; | |
353 } | |
354 #endif | |
355 | |
2348 | 356 |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
357 #ifdef CONFIG_PGM_ENCODER |
2344 | 358 AVCodec pgm_encoder = { |
359 "pgm", | |
360 CODEC_TYPE_VIDEO, | |
361 CODEC_ID_PGM, | |
362 sizeof(PNMContext), | |
363 common_init, | |
364 pnm_encode_frame, | |
365 NULL, //encode_end, | |
366 pnm_decode_frame, | |
4068 | 367 .pix_fmts= (enum PixelFormat[]){PIX_FMT_GRAY8, PIX_FMT_GRAY16BE, -1}, |
2344 | 368 }; |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
369 #endif // CONFIG_PGM_ENCODER |
2344 | 370 |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
371 #ifdef CONFIG_PGMYUV_ENCODER |
2344 | 372 AVCodec pgmyuv_encoder = { |
373 "pgmyuv", | |
374 CODEC_TYPE_VIDEO, | |
375 CODEC_ID_PGMYUV, | |
376 sizeof(PNMContext), | |
377 common_init, | |
378 pnm_encode_frame, | |
379 NULL, //encode_end, | |
380 pnm_decode_frame, | |
2967 | 381 .pix_fmts= (enum PixelFormat[]){PIX_FMT_YUV420P, -1}, |
2344 | 382 }; |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
383 #endif // CONFIG_PGMYUV_ENCODER |
2344 | 384 |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
385 #ifdef CONFIG_PPM_ENCODER |
2344 | 386 AVCodec ppm_encoder = { |
387 "ppm", | |
388 CODEC_TYPE_VIDEO, | |
389 CODEC_ID_PPM, | |
390 sizeof(PNMContext), | |
391 common_init, | |
392 pnm_encode_frame, | |
393 NULL, //encode_end, | |
394 pnm_decode_frame, | |
2967 | 395 .pix_fmts= (enum PixelFormat[]){PIX_FMT_RGB24, -1}, |
2344 | 396 }; |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
397 #endif // CONFIG_PPM_ENCODER |
2344 | 398 |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
399 #ifdef CONFIG_PBM_ENCODER |
2344 | 400 AVCodec pbm_encoder = { |
401 "pbm", | |
402 CODEC_TYPE_VIDEO, | |
403 CODEC_ID_PBM, | |
404 sizeof(PNMContext), | |
405 common_init, | |
406 pnm_encode_frame, | |
407 NULL, //encode_end, | |
408 pnm_decode_frame, | |
2967 | 409 .pix_fmts= (enum PixelFormat[]){PIX_FMT_MONOWHITE, -1}, |
2344 | 410 }; |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
411 #endif // CONFIG_PBM_ENCODER |
2344 | 412 |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
413 #ifdef CONFIG_PAM_ENCODER |
2344 | 414 AVCodec pam_encoder = { |
415 "pam", | |
416 CODEC_TYPE_VIDEO, | |
417 CODEC_ID_PAM, | |
418 sizeof(PNMContext), | |
419 common_init, | |
420 pam_encode_frame, | |
421 NULL, //encode_end, | |
2349 | 422 pnm_decode_frame, |
4494
ce643a22f049
Replace deprecated PIX_FMT names by the newer variants.
diego
parents:
4152
diff
changeset
|
423 .pix_fmts= (enum PixelFormat[]){PIX_FMT_RGB24, PIX_FMT_RGB32, PIX_FMT_GRAY8, PIX_FMT_MONOWHITE, -1}, |
2344 | 424 }; |
2661
b2846918585c
a few #ifdef CONFIG_X_ENCODER, patch by (Roine Gustafsson <roine users.sourceforge net]
michael
parents:
2453
diff
changeset
|
425 #endif // CONFIG_PAM_ENCODER |