changeset 1079:40e81416015d libavformat

sanity checks some might have been exploitable

--- a/rm.c	Fri May 12 15:13:51 2006 +0000
+++ b/rm.c	Sat May 13 11:37:56 2006 +0000
@@ -555,6 +555,12 @@
             st->codec->extradata_size= 0;
             rm->audio_framesize = st->codec->block_align;
             st->codec->block_align = coded_framesize;
+
+            if(rm->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h too large\n");
+                return -1;
+            }
+
             rm->audiobuf = av_malloc(rm->audio_framesize * sub_packet_h);
         } else if (!strcmp(buf, "cook")) {
             int codecdata_length, i;
@@ -562,6 +568,11 @@
             if (((version >> 16) & 0xff) == 5)
                 get_byte(pb);
             codecdata_length = get_be32(pb);
+            if(codecdata_length + FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)codecdata_length){
+                av_log(s, AV_LOG_ERROR, "codecdata_length too large\n");
+                return -1;
+            }
+
             st->codec->codec_id = CODEC_ID_COOK;
             st->codec->extradata_size= codecdata_length;
             st->codec->extradata= av_mallocz(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
@@ -569,6 +580,12 @@
                 ((uint8_t*)st->codec->extradata)[i] = get_byte(pb);
             rm->audio_framesize = st->codec->block_align;
             st->codec->block_align = rm->sub_packet_size;
+
+            if(rm->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h too large\n");
+                return -1;
+            }
+
             rm->audiobuf = av_malloc(rm->audio_framesize * sub_packet_h);
         } else {
             st->codec->codec_id = CODEC_ID_NONE;
@@ -715,6 +732,12 @@
                 get_be16(pb);
 
                 st->codec->extradata_size= codec_data_size - (url_ftell(pb) - codec_pos);
+
+                if(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)st->codec->extradata_size){
+                    //check is redundant as get_buffer() will catch this
+                    av_log(s, AV_LOG_ERROR, "st->codec->extradata_size too large\n");
+                    return -1;
+                }
                 st->codec->extradata= av_mallocz(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
                 get_buffer(pb, st->codec->extradata, st->codec->extradata_size);
 

--- a/sierravmd.c	Fri May 12 15:13:51 2006 +0000
+++ b/sierravmd.c	Sat May 13 11:37:56 2006 +0000
@@ -196,6 +196,10 @@
     vmd->frame_table = NULL;
     raw_frame_table_size = vmd->frame_count * 6;
     raw_frame_table = av_malloc(raw_frame_table_size);
+    if(vmd->frame_count * vmd->frames_per_block  >= UINT_MAX / sizeof(vmd_frame_t)){
+        av_log(s, AV_LOG_ERROR, "vmd->frame_count * vmd->frames_per_block too large\n");
+        return -1;
+    }
     vmd->frame_table = av_malloc(vmd->frame_count * vmd->frames_per_block * sizeof(vmd_frame_t));
     if (!raw_frame_table || !vmd->frame_table) {
         av_free(raw_frame_table);

--- a/smacker.c	Fri May 12 15:13:51 2006 +0000
+++ b/smacker.c	Sat May 13 11:37:56 2006 +0000
@@ -114,6 +114,13 @@
     for(i = 0; i < 7; i++)
         smk->audio[i] = get_le32(pb);
     smk->treesize = get_le32(pb);
+
+    if(smk->treesize >= UINT_MAX/4){ // smk->treesize + 16 must not overflow (this check is probably redundant)
+        av_log(s, AV_LOG_ERROR, "treesize too large\n");
+        return -1;
+    }
+
+//FIXME remove extradata "rebuilding"
     smk->mmap_size = get_le32(pb);
     smk->mclr_size = get_le32(pb);
     smk->full_size = get_le32(pb);

--- a/tta.c	Fri May 12 15:13:51 2006 +0000
+++ b/tta.c	Sat May 13 11:37:56 2006 +0000
@@ -50,13 +50,27 @@
     channels = get_le16(&s->pb);
     bps = get_le16(&s->pb);
     samplerate = get_le32(&s->pb);
+    if(samplerate <= 0 || samplerate > 1000000){
+        av_log(s, AV_LOG_ERROR, "nonsense samplerate\n");
+        return -1;
+    }
+
     datalen = get_le32(&s->pb);
+    if(datalen < 0){
+        av_log(s, AV_LOG_ERROR, "nonsense datalen\n");
+        return -1;
+    }
+
     url_fskip(&s->pb, 4); // header crc
 
     framelen = 1.04489795918367346939 * samplerate;
     c->totalframes = datalen / framelen + ((datalen % framelen) ? 1 : 0);
     c->currentframe = 0;
 
+    if(c->totalframes >= UINT_MAX/sizeof(uint32_t)){
+        av_log(s, AV_LOG_ERROR, "totalframes too large\n");
+        return -1;
+    }
     c->seektable = av_mallocz(sizeof(uint32_t)*c->totalframes);
     if (!c->seektable)
         return AVERROR_NOMEM;
@@ -76,6 +90,11 @@
     st->codec->bits_per_sample = bps;
 
     st->codec->extradata_size = url_ftell(&s->pb) - start;
+    if(st->codec->extradata_size+FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)st->codec->extradata_size){
+        //this check is redundant as get_buffer should fail
+        av_log(s, AV_LOG_ERROR, "extradata_size too large\n");
+        return -1;
+    }
     st->codec->extradata = av_mallocz(st->codec->extradata_size+FF_INPUT_BUFFER_PADDING_SIZE);
     url_fseek(&s->pb, start, SEEK_SET); // or SEEK_CUR and -size ? :)
     get_buffer(&s->pb, st->codec->extradata, st->codec->extradata_size);