# HG changeset patch # User michael # Date 1169301000 0 # Node ID 9cebff821565fe57b6bb1dbf6d5b2d75d52c5dd5 # Parent 05e932ddaaa99a602f917f6b723fe6188bf119c5 checking bitstream values and other related changes some of these might have been exploitable diff -r 05e932ddaaa9 -r 9cebff821565 h264.c --- a/h264.c Fri Jan 19 22:12:59 2007 +0000 +++ b/h264.c Sat Jan 20 13:50:00 2007 +0000 @@ -122,7 +122,7 @@ int pic_order_present; ///< pic_order_present_flag int slice_group_count; ///< num_slice_groups_minus1 + 1 int mb_slice_group_map_type; - int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1 + unsigned int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1 int weighted_pred; ///< weighted_pred_flag int weighted_bipred_idc; int init_qp; ///< pic_init_qp_minus26 + 26 @@ -274,7 +274,7 @@ int mb_field_decoding_flag; int mb_mbaff; ///< mb_aff_frame && mb_field_decoding_flag - int sub_mb_type[4]; + unsigned int sub_mb_type[4]; //POC stuff int poc_lsb; @@ -325,7 +325,7 @@ /** * num_ref_idx_l0/1_active_minus1 + 1 */ - int ref_count[2]; ///< counts frames or fields, depending on current mb mode + unsigned int ref_count[2]; ///< counts frames or fields, depending on current mb mode Picture *short_ref[32]; Picture *long_ref[32]; Picture default_ref_list[2][32]; @@ -349,6 +349,7 @@ GetBitContext *inter_gb_ptr; DECLARE_ALIGNED_8(DCTELEM, mb[16*24]); + DCTELEM mb_padding[256]; ///< as mb is addressed by scantable[i] and scantable is uint8_t we can either check that i is not to large or ensure that there is some unused stuff after mb /** * Cabac @@ -1398,7 +1399,7 @@ const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy]; const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy]; const int is_b8x8 = IS_8X8(*mb_type); - int sub_mb_type; + unsigned int sub_mb_type; int i8, i4; #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM) @@ -3916,8 +3917,8 @@ int pred= h->curr_pic_num; for(index=0; ; index++){ - int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb); - int pic_id; + unsigned int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb); + unsigned int pic_id; int i; Picture *ref = NULL; @@ -3931,7 +3932,7 @@ if(reordering_of_pic_nums_idc<3){ if(reordering_of_pic_nums_idc<2){ - const int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1; + const unsigned int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1; if(abs_diff_pic_num >= h->max_pic_num){ av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n"); @@ -3953,6 +3954,10 @@ ref->pic_id= ref->frame_num; }else{ pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx + if(pic_id>31){ + av_log(h->s.avctx, AV_LOG_ERROR, "long_term_pic_idx overflow\n"); + return -1; + } ref = h->long_ref[pic_id]; if(ref){ ref->pic_id= pic_id; @@ -4358,14 +4363,15 @@ }*/ } if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){ - h->mmco[i].long_index= get_ue_golomb(&s->gb); - if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ h->mmco[i].long_index >= 16){ + unsigned int long_index= get_ue_golomb(&s->gb); + if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ long_index >= 16){ av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode); return -1; } + h->mmco[i].long_index= long_index; } - if(opcode > MMCO_LONG){ + if(opcode > (unsigned)MMCO_LONG){ av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode); return -1; } @@ -4483,11 +4489,11 @@ */ static int decode_slice_header(H264Context *h){ MpegEncContext * const s = &h->s; - int first_mb_in_slice; + unsigned int first_mb_in_slice; unsigned int pps_id; int num_ref_idx_active_override_flag; static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE}; - int slice_type; + unsigned int slice_type, tmp; int default_ref_list_done = 0; s->current_picture.reference= h->nal_ref_idc != 0; @@ -4640,12 +4646,15 @@ h->mb_aff_frame = h->sps.mb_aff; } } - + assert(s->mb_num == s->mb_width * s->mb_height); + if(first_mb_in_slice << h->mb_aff_frame >= s->mb_num || + first_mb_in_slice >= s->mb_num){ + av_log(h->s.avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n"); + return -1; + } s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width; s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame; - if(s->mb_y >= s->mb_height){ - return -1; - } + assert(s->mb_y < s->mb_height); if(s->picture_structure==PICT_FRAME){ h->curr_pic_num= h->frame_num; @@ -4699,6 +4708,7 @@ if(h->ref_count[0] > 32 || h->ref_count[1] > 32){ av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); + h->ref_count[0]= h->ref_count[1]= 1; return -1; } } @@ -4725,15 +4735,22 @@ if(FRAME_MBAFF) fill_mbaff_ref_list(h); - if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ) - h->cabac_init_idc = get_ue_golomb(&s->gb); + if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ){ + tmp = get_ue_golomb(&s->gb); + if(tmp > 2){ + av_log(s->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n"); + return -1; + } + h->cabac_init_idc= tmp; + } h->last_qscale_diff = 0; - s->qscale = h->pps.init_qp + get_se_golomb(&s->gb); - if(s->qscale<0 || s->qscale>51){ - av_log(s->avctx, AV_LOG_ERROR, "QP %d out of range\n", s->qscale); + tmp = h->pps.init_qp + get_se_golomb(&s->gb); + if(tmp>51){ + av_log(s->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp); return -1; } + s->qscale= tmp; h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale); //FIXME qscale / qp ... stuff if(h->slice_type == SP_TYPE){ @@ -4747,7 +4764,12 @@ h->slice_alpha_c0_offset = 0; h->slice_beta_offset = 0; if( h->pps.deblocking_filter_parameters_present ) { - h->deblocking_filter= get_ue_golomb(&s->gb); + tmp= get_ue_golomb(&s->gb); + if(tmp > 2){ + av_log(s->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp); + return -1; + } + h->deblocking_filter= tmp; if(h->deblocking_filter < 2) h->deblocking_filter^= 1; // 1<->0 @@ -5044,7 +5066,8 @@ static int decode_mb_cavlc(H264Context *h){ MpegEncContext * const s = &h->s; const int mb_xy= s->mb_x + s->mb_y*s->mb_stride; - int mb_type, partition_count, cbp; + int partition_count; + unsigned int mb_type, cbp; int dct8x8_allowed= h->pps.transform_8x8_mode; s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong? @@ -5158,6 +5181,7 @@ //mb_pred if(IS_INTRA(mb_type)){ + int pred_mode; // init_top_left_availability(h); if(IS_INTRA4x4(mb_type)){ int i; @@ -5189,11 +5213,11 @@ if(h->intra16x16_pred_mode < 0) return -1; } - h->chroma_pred_mode= get_ue_golomb(&s->gb); - - h->chroma_pred_mode= check_intra_pred_mode(h, h->chroma_pred_mode); - if(h->chroma_pred_mode < 0) + + pred_mode= check_intra_pred_mode(h, get_ue_golomb(&s->gb)); + if(pred_mode < 0) return -1; + h->chroma_pred_mode= pred_mode; }else if(partition_count==4){ int i, j, sub_partition_count[4], list, ref[2][4]; @@ -5201,7 +5225,7 @@ for(i=0; i<4; i++){ h->sub_mb_type[i]= get_ue_golomb(&s->gb); if(h->sub_mb_type[i] >=13){ - av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); + av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); return -1; } sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; @@ -5220,7 +5244,7 @@ for(i=0; i<4; i++){ h->sub_mb_type[i]= get_ue_golomb(&s->gb); if(h->sub_mb_type[i] >=4){ - av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); + av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); return -1; } sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; @@ -5234,7 +5258,12 @@ for(i=0; i<4; i++){ if(IS_DIRECT(h->sub_mb_type[i])) continue; if(IS_DIR(h->sub_mb_type[i], 0, list)){ - ref[list][i] = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip? + unsigned int tmp = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip? + if(tmp>=ref_count){ + av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", tmp); + return -1; + } + ref[list][i]= tmp; }else{ //FIXME ref[list][i] = -1; @@ -5303,7 +5332,11 @@ for(list=0; list<2; list++){ if(h->ref_count[list]>0){ if(IS_DIR(mb_type, 0, list)){ - const int val= get_te0_golomb(&s->gb, h->ref_count[list]); + unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); + if(val >= h->ref_count[list]){ + av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); + return -1; + } fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1); }else fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1); @@ -5326,7 +5359,11 @@ if(h->ref_count[list]>0){ for(i=0; i<2; i++){ if(IS_DIR(mb_type, i, list)){ - const int val= get_te0_golomb(&s->gb, h->ref_count[list]); + unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); + if(val >= h->ref_count[list]){ + av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); + return -1; + } fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1); }else fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); @@ -5352,7 +5389,11 @@ if(h->ref_count[list]>0){ for(i=0; i<2; i++){ if(IS_DIR(mb_type, i, list)){ //FIXME optimize - const int val= get_te0_golomb(&s->gb, h->ref_count[list]); + unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); + if(val >= h->ref_count[list]){ + av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); + return -1; + } fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1); }else fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); @@ -5381,7 +5422,7 @@ if(!IS_INTRA16x16(mb_type)){ cbp= get_ue_golomb(&s->gb); if(cbp > 47){ - av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%d) at %d %d\n", cbp, s->mb_x, s->mb_y); + av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%u) at %d %d\n", cbp, s->mb_x, s->mb_y); return -1; } @@ -5863,6 +5904,10 @@ ctx = 4; else ctx = 5; + if(ref >= 32 /*h->ref_list[list]*/){ + av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n"); + return 0; //FIXME we should return -1 and check the return everywhere + } } return ref; } @@ -5896,6 +5941,10 @@ while( get_cabac_bypass( &h->cabac ) ) { mvd += 1 << k; k++; + if(k>24){ + av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_mvd\n"); + return INT_MIN; + } } while( k-- ) { if( get_cabac_bypass( &h->cabac ) ) @@ -6285,7 +6334,7 @@ fill_caches(h, mb_type, 0); if( IS_INTRA( mb_type ) ) { - int i; + int i, pred_mode; if( IS_INTRA4x4( mb_type ) ) { if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) { mb_type |= MB_TYPE_8x8DCT; @@ -6309,10 +6358,11 @@ if( h->intra16x16_pred_mode < 0 ) return -1; } h->chroma_pred_mode_table[mb_xy] = - h->chroma_pred_mode = decode_cabac_mb_chroma_pre_mode( h ); - - h->chroma_pred_mode= check_intra_pred_mode( h, h->chroma_pred_mode ); - if( h->chroma_pred_mode < 0 ) return -1; + pred_mode = decode_cabac_mb_chroma_pre_mode( h ); + + pred_mode= check_intra_pred_mode( h, pred_mode ); + if( pred_mode < 0 ) return -1; + h->chroma_pred_mode= pred_mode; } else if( partition_count == 4 ) { int i, j, sub_partition_count[4], list, ref[2][4]; @@ -7561,7 +7611,8 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){ MpegEncContext * const s = &h->s; - int aspect_ratio_info_present_flag, aspect_ratio_idc; + int aspect_ratio_info_present_flag; + unsigned int aspect_ratio_idc; int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag; aspect_ratio_info_present_flag= get_bits1(&s->gb); @@ -7683,7 +7734,7 @@ static inline int decode_seq_parameter_set(H264Context *h){ MpegEncContext * const s = &h->s; int profile_idc, level_idc; - unsigned int sps_id; + unsigned int sps_id, tmp, mb_width, mb_height; int i; SPS *sps; @@ -7725,26 +7776,36 @@ sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb); sps->offset_for_non_ref_pic= get_se_golomb(&s->gb); sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb); - sps->poc_cycle_length= get_ue_golomb(&s->gb); + tmp= get_ue_golomb(&s->gb); + + if(tmp >= sizeof(sps->offset_for_ref_frame) / sizeof(sps->offset_for_ref_frame[0])){ + av_log(h->s.avctx, AV_LOG_ERROR, "poc_cycle_length overflow %u\n", tmp); + return -1; + } + sps->poc_cycle_length= tmp; for(i=0; ipoc_cycle_length; i++) sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb); - } - if(sps->poc_type > 2){ + }else if(sps->poc_type != 2){ av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type); return -1; } - sps->ref_frame_count= get_ue_golomb(&s->gb); - if(sps->ref_frame_count > MAX_PICTURE_COUNT-2){ + tmp= get_ue_golomb(&s->gb); + if(tmp > MAX_PICTURE_COUNT-2){ av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n"); } + sps->ref_frame_count= tmp; sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb); - sps->mb_width= get_ue_golomb(&s->gb) + 1; - sps->mb_height= get_ue_golomb(&s->gb) + 1; - if((unsigned)sps->mb_width >= INT_MAX/16 || (unsigned)sps->mb_height >= INT_MAX/16 || - avcodec_check_dimensions(NULL, 16*sps->mb_width, 16*sps->mb_height)) + mb_width= get_ue_golomb(&s->gb) + 1; + mb_height= get_ue_golomb(&s->gb) + 1; + if(mb_width >= INT_MAX/16 || mb_height >= INT_MAX/16 || + avcodec_check_dimensions(NULL, 16*mb_width, 16*mb_height)){ + av_log(h->s.avctx, AV_LOG_ERROR, "mb_width/height overflow\n"); return -1; + } + sps->mb_width = mb_width; + sps->mb_height= mb_height; sps->frame_mbs_only_flag= get_bits1(&s->gb); if(!sps->frame_mbs_only_flag) @@ -7799,7 +7860,7 @@ static inline int decode_picture_parameter_set(H264Context *h, int bit_length){ MpegEncContext * const s = &h->s; - unsigned int pps_id= get_ue_golomb(&s->gb); + unsigned int tmp, pps_id= get_ue_golomb(&s->gb); PPS *pps; if(pps_id>=MAX_PPS_COUNT){ @@ -7808,7 +7869,13 @@ } pps = &h->pps_buffer[pps_id]; - pps->sps_id= get_ue_golomb(&s->gb); + tmp= get_ue_golomb(&s->gb); + if(tmp>=MAX_SPS_COUNT){ + av_log(h->s.avctx, AV_LOG_ERROR, "sps_id out of range\n"); + return -1; + } + pps->sps_id= tmp; + pps->cabac= get_bits1(&s->gb); pps->pic_order_present= get_bits1(&s->gb); pps->slice_group_count= get_ue_golomb(&s->gb) + 1; @@ -7853,6 +7920,7 @@ pps->ref_count[1]= get_ue_golomb(&s->gb) + 1; if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){ av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n"); + pps->ref_count[0]= pps->ref_count[1]= 1; return -1; }