Mercurial > libavcodec.hg

comparison shorten.c @ 11964:f092b80c0da2 libavcodec

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
shorten: remove VLA and check for buffer overflow
author mru
date Sat, 26 Jun 2010 14:34:21 +0000
parents 7dd2a45249a9
children 8b28e74de2c0
comparison
equal deleted inserted replaced
11963:74c0478534cf 11964:f092b80c0da2
81 int min_framesize, max_framesize; 81 int min_framesize, max_framesize;
82 int channels; 82 int channels;
83 83
84 int32_t *decoded[MAX_CHANNELS]; 84 int32_t *decoded[MAX_CHANNELS];
85 int32_t *offset[MAX_CHANNELS]; 85 int32_t *offset[MAX_CHANNELS];
86 int *coeffs;
86 uint8_t *bitstream; 87 uint8_t *bitstream;
87 int bitstream_size; 88 int bitstream_size;
88 int bitstream_index; 89 int bitstream_index;
89 unsigned int allocated_bitstream_size; 90 unsigned int allocated_bitstream_size;
90 int header_size; 91 int header_size;
110 } 111 }
111 112
112 static int allocate_buffers(ShortenContext *s) 113 static int allocate_buffers(ShortenContext *s)
113 { 114 {
114 int i, chan; 115 int i, chan;
116 int *coeffs;
117
115 for (chan=0; chan<s->channels; chan++) { 118 for (chan=0; chan<s->channels; chan++) {
116 if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){ 119 if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
117 av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n"); 120 av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
118 return -1; 121 return -1;
119 } 122 }
127 s->decoded[chan] = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap)); 130 s->decoded[chan] = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
128 for (i=0; i<s->nwrap; i++) 131 for (i=0; i<s->nwrap; i++)
129 s->decoded[chan][i] = 0; 132 s->decoded[chan][i] = 0;
130 s->decoded[chan] += s->nwrap; 133 s->decoded[chan] += s->nwrap;
131 } 134 }
135
136 coeffs = av_realloc(s->coeffs, s->nwrap * sizeof(*s->coeffs));
137 if (!coeffs)
138 return AVERROR(ENOMEM);
139 s->coeffs = coeffs;
140
132 return 0; 141 return 0;
133 } 142 }
134 143
135 144
136 static inline unsigned int get_uint(ShortenContext *s, int k) 145 static inline unsigned int get_uint(ShortenContext *s, int k)
251 } 260 }
252 261
253 static void decode_subframe_lpc(ShortenContext *s, int channel, int residual_size, int pred_order) 262 static void decode_subframe_lpc(ShortenContext *s, int channel, int residual_size, int pred_order)
254 { 263 {
255 int sum, i, j; 264 int sum, i, j;
256 int coeffs[pred_order]; 265 int *coeffs = s->coeffs;
257 266
258 for (i=0; i<pred_order; i++) 267 for (i=0; i<pred_order; i++)
259 coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT); 268 coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
260 269
261 for (i=0; i < s->blocksize; i++) { 270 for (i=0; i < s->blocksize; i++) {
425 + s->decoded[channel][i-3]; 434 + s->decoded[channel][i-3];
426 break; 435 break;
427 case FN_QLPC: 436 case FN_QLPC:
428 { 437 {
429 int pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE); 438 int pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
439 if (pred_order > s->nwrap) {
440 av_log(avctx, AV_LOG_ERROR,
441 "invalid pred_order %d\n",
442 pred_order);
443 return -1;
444 }
430 for (i=0; i<pred_order; i++) 445 for (i=0; i<pred_order; i++)
431 s->decoded[channel][i - pred_order] -= coffset; 446 s->decoded[channel][i - pred_order] -= coffset;
432 decode_subframe_lpc(s, channel, residual_size, pred_order); 447 decode_subframe_lpc(s, channel, residual_size, pred_order);
433 if (coffset != 0) 448 if (coffset != 0)
434 for (i=0; i < s->blocksize; i++) 449 for (i=0; i < s->blocksize; i++)
513 s->decoded[i] -= s->nwrap; 528 s->decoded[i] -= s->nwrap;
514 av_freep(&s->decoded[i]); 529 av_freep(&s->decoded[i]);
515 av_freep(&s->offset[i]); 530 av_freep(&s->offset[i]);
516 } 531 }
517 av_freep(&s->bitstream); 532 av_freep(&s->bitstream);
533 av_freep(&s->coeffs);
518 return 0; 534 return 0;
519 } 535 }
520 536
521 static void shorten_flush(AVCodecContext *avctx){ 537 static void shorten_flush(AVCodecContext *avctx){
522 ShortenContext *s = avctx->priv_data; 538 ShortenContext *s = avctx->priv_data;