diff shorten.c @ 11964:f092b80c0da2 libavcodec

shorten: remove VLA and check for buffer overflow

--- a/shorten.c	Sat Jun 26 14:34:18 2010 +0000
+++ b/shorten.c	Sat Jun 26 14:34:21 2010 +0000
@@ -83,6 +83,7 @@
 
     int32_t *decoded[MAX_CHANNELS];
     int32_t *offset[MAX_CHANNELS];
+    int *coeffs;
     uint8_t *bitstream;
     int bitstream_size;
     int bitstream_index;
@@ -112,6 +113,8 @@
 static int allocate_buffers(ShortenContext *s)
 {
     int i, chan;
+    int *coeffs;
+
     for (chan=0; chan<s->channels; chan++) {
         if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
             av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
@@ -129,6 +132,12 @@
             s->decoded[chan][i] = 0;
         s->decoded[chan] += s->nwrap;
     }
+
+    coeffs = av_realloc(s->coeffs, s->nwrap * sizeof(*s->coeffs));
+    if (!coeffs)
+        return AVERROR(ENOMEM);
+    s->coeffs = coeffs;
+
     return 0;
 }
 
@@ -253,7 +262,7 @@
 static void decode_subframe_lpc(ShortenContext *s, int channel, int residual_size, int pred_order)
 {
     int sum, i, j;
-    int coeffs[pred_order];
+    int *coeffs = s->coeffs;
 
     for (i=0; i<pred_order; i++)
         coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
@@ -427,6 +436,12 @@
                         case FN_QLPC:
                             {
                                 int pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
+                                if (pred_order > s->nwrap) {
+                                    av_log(avctx, AV_LOG_ERROR,
+                                           "invalid pred_order %d\n",
+                                           pred_order);
+                                    return -1;
+                                }
                                 for (i=0; i<pred_order; i++)
                                     s->decoded[channel][i - pred_order] -= coffset;
                                 decode_subframe_lpc(s, channel, residual_size, pred_order);
@@ -515,6 +530,7 @@
         av_freep(&s->offset[i]);
     }
     av_freep(&s->bitstream);
+    av_freep(&s->coeffs);
     return 0;
 }