Mercurial > libavcodec.hg
comparison h264.c @ 4365:9cebff821565 libavcodec
checking bitstream values and other related changes
some of these might have been exploitable
author | michael |
---|---|
date | Sat, 20 Jan 2007 13:50:00 +0000 |
parents | 05e932ddaaa9 |
children | 3451831c6962 |
comparison
equal
deleted
inserted
replaced
4364:05e932ddaaa9 | 4365:9cebff821565 |
---|---|
120 unsigned int sps_id; | 120 unsigned int sps_id; |
121 int cabac; ///< entropy_coding_mode_flag | 121 int cabac; ///< entropy_coding_mode_flag |
122 int pic_order_present; ///< pic_order_present_flag | 122 int pic_order_present; ///< pic_order_present_flag |
123 int slice_group_count; ///< num_slice_groups_minus1 + 1 | 123 int slice_group_count; ///< num_slice_groups_minus1 + 1 |
124 int mb_slice_group_map_type; | 124 int mb_slice_group_map_type; |
125 int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1 | 125 unsigned int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1 |
126 int weighted_pred; ///< weighted_pred_flag | 126 int weighted_pred; ///< weighted_pred_flag |
127 int weighted_bipred_idc; | 127 int weighted_bipred_idc; |
128 int init_qp; ///< pic_init_qp_minus26 + 26 | 128 int init_qp; ///< pic_init_qp_minus26 + 26 |
129 int init_qs; ///< pic_init_qs_minus26 + 26 | 129 int init_qs; ///< pic_init_qs_minus26 + 26 |
130 int chroma_qp_index_offset; | 130 int chroma_qp_index_offset; |
272 //interlacing specific flags | 272 //interlacing specific flags |
273 int mb_aff_frame; | 273 int mb_aff_frame; |
274 int mb_field_decoding_flag; | 274 int mb_field_decoding_flag; |
275 int mb_mbaff; ///< mb_aff_frame && mb_field_decoding_flag | 275 int mb_mbaff; ///< mb_aff_frame && mb_field_decoding_flag |
276 | 276 |
277 int sub_mb_type[4]; | 277 unsigned int sub_mb_type[4]; |
278 | 278 |
279 //POC stuff | 279 //POC stuff |
280 int poc_lsb; | 280 int poc_lsb; |
281 int poc_msb; | 281 int poc_msb; |
282 int delta_poc_bottom; | 282 int delta_poc_bottom; |
323 int map_col_to_list0_field[2][32]; | 323 int map_col_to_list0_field[2][32]; |
324 | 324 |
325 /** | 325 /** |
326 * num_ref_idx_l0/1_active_minus1 + 1 | 326 * num_ref_idx_l0/1_active_minus1 + 1 |
327 */ | 327 */ |
328 int ref_count[2]; ///< counts frames or fields, depending on current mb mode | 328 unsigned int ref_count[2]; ///< counts frames or fields, depending on current mb mode |
329 Picture *short_ref[32]; | 329 Picture *short_ref[32]; |
330 Picture *long_ref[32]; | 330 Picture *long_ref[32]; |
331 Picture default_ref_list[2][32]; | 331 Picture default_ref_list[2][32]; |
332 Picture ref_list[2][48]; ///< 0..15: frame refs, 16..47: mbaff field refs | 332 Picture ref_list[2][48]; ///< 0..15: frame refs, 16..47: mbaff field refs |
333 Picture *delayed_pic[16]; //FIXME size? | 333 Picture *delayed_pic[16]; //FIXME size? |
347 GetBitContext inter_gb; | 347 GetBitContext inter_gb; |
348 GetBitContext *intra_gb_ptr; | 348 GetBitContext *intra_gb_ptr; |
349 GetBitContext *inter_gb_ptr; | 349 GetBitContext *inter_gb_ptr; |
350 | 350 |
351 DECLARE_ALIGNED_8(DCTELEM, mb[16*24]); | 351 DECLARE_ALIGNED_8(DCTELEM, mb[16*24]); |
352 DCTELEM mb_padding[256]; ///< as mb is addressed by scantable[i] and scantable is uint8_t we can either check that i is not to large or ensure that there is some unused stuff after mb | |
352 | 353 |
353 /** | 354 /** |
354 * Cabac | 355 * Cabac |
355 */ | 356 */ |
356 CABACContext cabac; | 357 CABACContext cabac; |
1396 const int16_t (*l1mv0)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[0][b4_xy]; | 1397 const int16_t (*l1mv0)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[0][b4_xy]; |
1397 const int16_t (*l1mv1)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[1][b4_xy]; | 1398 const int16_t (*l1mv1)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[1][b4_xy]; |
1398 const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy]; | 1399 const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy]; |
1399 const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy]; | 1400 const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy]; |
1400 const int is_b8x8 = IS_8X8(*mb_type); | 1401 const int is_b8x8 = IS_8X8(*mb_type); |
1401 int sub_mb_type; | 1402 unsigned int sub_mb_type; |
1402 int i8, i4; | 1403 int i8, i4; |
1403 | 1404 |
1404 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM) | 1405 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM) |
1405 if(IS_8X8(mb_type_col) && !h->sps.direct_8x8_inference_flag){ | 1406 if(IS_8X8(mb_type_col) && !h->sps.direct_8x8_inference_flag){ |
1406 /* FIXME save sub mb types from previous frames (or derive from MVs) | 1407 /* FIXME save sub mb types from previous frames (or derive from MVs) |
3914 | 3915 |
3915 if(get_bits1(&s->gb)){ | 3916 if(get_bits1(&s->gb)){ |
3916 int pred= h->curr_pic_num; | 3917 int pred= h->curr_pic_num; |
3917 | 3918 |
3918 for(index=0; ; index++){ | 3919 for(index=0; ; index++){ |
3919 int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb); | 3920 unsigned int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb); |
3920 int pic_id; | 3921 unsigned int pic_id; |
3921 int i; | 3922 int i; |
3922 Picture *ref = NULL; | 3923 Picture *ref = NULL; |
3923 | 3924 |
3924 if(reordering_of_pic_nums_idc==3) | 3925 if(reordering_of_pic_nums_idc==3) |
3925 break; | 3926 break; |
3929 return -1; | 3930 return -1; |
3930 } | 3931 } |
3931 | 3932 |
3932 if(reordering_of_pic_nums_idc<3){ | 3933 if(reordering_of_pic_nums_idc<3){ |
3933 if(reordering_of_pic_nums_idc<2){ | 3934 if(reordering_of_pic_nums_idc<2){ |
3934 const int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1; | 3935 const unsigned int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1; |
3935 | 3936 |
3936 if(abs_diff_pic_num >= h->max_pic_num){ | 3937 if(abs_diff_pic_num >= h->max_pic_num){ |
3937 av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n"); | 3938 av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n"); |
3938 return -1; | 3939 return -1; |
3939 } | 3940 } |
3951 } | 3952 } |
3952 if(i>=0) | 3953 if(i>=0) |
3953 ref->pic_id= ref->frame_num; | 3954 ref->pic_id= ref->frame_num; |
3954 }else{ | 3955 }else{ |
3955 pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx | 3956 pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx |
3957 if(pic_id>31){ | |
3958 av_log(h->s.avctx, AV_LOG_ERROR, "long_term_pic_idx overflow\n"); | |
3959 return -1; | |
3960 } | |
3956 ref = h->long_ref[pic_id]; | 3961 ref = h->long_ref[pic_id]; |
3957 if(ref){ | 3962 if(ref){ |
3958 ref->pic_id= pic_id; | 3963 ref->pic_id= pic_id; |
3959 assert(ref->reference == 3); | 3964 assert(ref->reference == 3); |
3960 assert(ref->long_ref); | 3965 assert(ref->long_ref); |
4356 av_log(s->avctx, AV_LOG_ERROR, "illegal short ref in memory management control operation %d\n", mmco); | 4361 av_log(s->avctx, AV_LOG_ERROR, "illegal short ref in memory management control operation %d\n", mmco); |
4357 return -1; | 4362 return -1; |
4358 }*/ | 4363 }*/ |
4359 } | 4364 } |
4360 if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){ | 4365 if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){ |
4361 h->mmco[i].long_index= get_ue_golomb(&s->gb); | 4366 unsigned int long_index= get_ue_golomb(&s->gb); |
4362 if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ h->mmco[i].long_index >= 16){ | 4367 if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ long_index >= 16){ |
4363 av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode); | 4368 av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode); |
4364 return -1; | 4369 return -1; |
4365 } | 4370 } |
4366 } | 4371 h->mmco[i].long_index= long_index; |
4367 | 4372 } |
4368 if(opcode > MMCO_LONG){ | 4373 |
4374 if(opcode > (unsigned)MMCO_LONG){ | |
4369 av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode); | 4375 av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode); |
4370 return -1; | 4376 return -1; |
4371 } | 4377 } |
4372 if(opcode == MMCO_END) | 4378 if(opcode == MMCO_END) |
4373 break; | 4379 break; |
4481 * decodes a slice header. | 4487 * decodes a slice header. |
4482 * this will allso call MPV_common_init() and frame_start() as needed | 4488 * this will allso call MPV_common_init() and frame_start() as needed |
4483 */ | 4489 */ |
4484 static int decode_slice_header(H264Context *h){ | 4490 static int decode_slice_header(H264Context *h){ |
4485 MpegEncContext * const s = &h->s; | 4491 MpegEncContext * const s = &h->s; |
4486 int first_mb_in_slice; | 4492 unsigned int first_mb_in_slice; |
4487 unsigned int pps_id; | 4493 unsigned int pps_id; |
4488 int num_ref_idx_active_override_flag; | 4494 int num_ref_idx_active_override_flag; |
4489 static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE}; | 4495 static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE}; |
4490 int slice_type; | 4496 unsigned int slice_type, tmp; |
4491 int default_ref_list_done = 0; | 4497 int default_ref_list_done = 0; |
4492 | 4498 |
4493 s->current_picture.reference= h->nal_ref_idc != 0; | 4499 s->current_picture.reference= h->nal_ref_idc != 0; |
4494 s->dropable= h->nal_ref_idc == 0; | 4500 s->dropable= h->nal_ref_idc == 0; |
4495 | 4501 |
4638 } else { | 4644 } else { |
4639 s->picture_structure= PICT_FRAME; | 4645 s->picture_structure= PICT_FRAME; |
4640 h->mb_aff_frame = h->sps.mb_aff; | 4646 h->mb_aff_frame = h->sps.mb_aff; |
4641 } | 4647 } |
4642 } | 4648 } |
4643 | 4649 assert(s->mb_num == s->mb_width * s->mb_height); |
4650 if(first_mb_in_slice << h->mb_aff_frame >= s->mb_num || | |
4651 first_mb_in_slice >= s->mb_num){ | |
4652 av_log(h->s.avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n"); | |
4653 return -1; | |
4654 } | |
4644 s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width; | 4655 s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width; |
4645 s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame; | 4656 s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame; |
4646 if(s->mb_y >= s->mb_height){ | 4657 assert(s->mb_y < s->mb_height); |
4647 return -1; | |
4648 } | |
4649 | 4658 |
4650 if(s->picture_structure==PICT_FRAME){ | 4659 if(s->picture_structure==PICT_FRAME){ |
4651 h->curr_pic_num= h->frame_num; | 4660 h->curr_pic_num= h->frame_num; |
4652 h->max_pic_num= 1<< h->sps.log2_max_frame_num; | 4661 h->max_pic_num= 1<< h->sps.log2_max_frame_num; |
4653 }else{ | 4662 }else{ |
4697 if(h->slice_type==B_TYPE) | 4706 if(h->slice_type==B_TYPE) |
4698 h->ref_count[1]= get_ue_golomb(&s->gb) + 1; | 4707 h->ref_count[1]= get_ue_golomb(&s->gb) + 1; |
4699 | 4708 |
4700 if(h->ref_count[0] > 32 || h->ref_count[1] > 32){ | 4709 if(h->ref_count[0] > 32 || h->ref_count[1] > 32){ |
4701 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); | 4710 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); |
4711 h->ref_count[0]= h->ref_count[1]= 1; | |
4702 return -1; | 4712 return -1; |
4703 } | 4713 } |
4704 } | 4714 } |
4705 } | 4715 } |
4706 | 4716 |
4723 decode_ref_pic_marking(h); | 4733 decode_ref_pic_marking(h); |
4724 | 4734 |
4725 if(FRAME_MBAFF) | 4735 if(FRAME_MBAFF) |
4726 fill_mbaff_ref_list(h); | 4736 fill_mbaff_ref_list(h); |
4727 | 4737 |
4728 if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ) | 4738 if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ){ |
4729 h->cabac_init_idc = get_ue_golomb(&s->gb); | 4739 tmp = get_ue_golomb(&s->gb); |
4740 if(tmp > 2){ | |
4741 av_log(s->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n"); | |
4742 return -1; | |
4743 } | |
4744 h->cabac_init_idc= tmp; | |
4745 } | |
4730 | 4746 |
4731 h->last_qscale_diff = 0; | 4747 h->last_qscale_diff = 0; |
4732 s->qscale = h->pps.init_qp + get_se_golomb(&s->gb); | 4748 tmp = h->pps.init_qp + get_se_golomb(&s->gb); |
4733 if(s->qscale<0 || s->qscale>51){ | 4749 if(tmp>51){ |
4734 av_log(s->avctx, AV_LOG_ERROR, "QP %d out of range\n", s->qscale); | 4750 av_log(s->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp); |
4735 return -1; | 4751 return -1; |
4736 } | 4752 } |
4753 s->qscale= tmp; | |
4737 h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale); | 4754 h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale); |
4738 //FIXME qscale / qp ... stuff | 4755 //FIXME qscale / qp ... stuff |
4739 if(h->slice_type == SP_TYPE){ | 4756 if(h->slice_type == SP_TYPE){ |
4740 get_bits1(&s->gb); /* sp_for_switch_flag */ | 4757 get_bits1(&s->gb); /* sp_for_switch_flag */ |
4741 } | 4758 } |
4745 | 4762 |
4746 h->deblocking_filter = 1; | 4763 h->deblocking_filter = 1; |
4747 h->slice_alpha_c0_offset = 0; | 4764 h->slice_alpha_c0_offset = 0; |
4748 h->slice_beta_offset = 0; | 4765 h->slice_beta_offset = 0; |
4749 if( h->pps.deblocking_filter_parameters_present ) { | 4766 if( h->pps.deblocking_filter_parameters_present ) { |
4750 h->deblocking_filter= get_ue_golomb(&s->gb); | 4767 tmp= get_ue_golomb(&s->gb); |
4768 if(tmp > 2){ | |
4769 av_log(s->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp); | |
4770 return -1; | |
4771 } | |
4772 h->deblocking_filter= tmp; | |
4751 if(h->deblocking_filter < 2) | 4773 if(h->deblocking_filter < 2) |
4752 h->deblocking_filter^= 1; // 1<->0 | 4774 h->deblocking_filter^= 1; // 1<->0 |
4753 | 4775 |
4754 if( h->deblocking_filter ) { | 4776 if( h->deblocking_filter ) { |
4755 h->slice_alpha_c0_offset = get_se_golomb(&s->gb) << 1; | 4777 h->slice_alpha_c0_offset = get_se_golomb(&s->gb) << 1; |
5042 * @returns 0 if ok, AC_ERROR / DC_ERROR / MV_ERROR if an error is noticed | 5064 * @returns 0 if ok, AC_ERROR / DC_ERROR / MV_ERROR if an error is noticed |
5043 */ | 5065 */ |
5044 static int decode_mb_cavlc(H264Context *h){ | 5066 static int decode_mb_cavlc(H264Context *h){ |
5045 MpegEncContext * const s = &h->s; | 5067 MpegEncContext * const s = &h->s; |
5046 const int mb_xy= s->mb_x + s->mb_y*s->mb_stride; | 5068 const int mb_xy= s->mb_x + s->mb_y*s->mb_stride; |
5047 int mb_type, partition_count, cbp; | 5069 int partition_count; |
5070 unsigned int mb_type, cbp; | |
5048 int dct8x8_allowed= h->pps.transform_8x8_mode; | 5071 int dct8x8_allowed= h->pps.transform_8x8_mode; |
5049 | 5072 |
5050 s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong? | 5073 s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong? |
5051 | 5074 |
5052 tprintf("pic:%d mb:%d/%d\n", h->frame_num, s->mb_x, s->mb_y); | 5075 tprintf("pic:%d mb:%d/%d\n", h->frame_num, s->mb_x, s->mb_y); |
5156 | 5179 |
5157 fill_caches(h, mb_type, 0); | 5180 fill_caches(h, mb_type, 0); |
5158 | 5181 |
5159 //mb_pred | 5182 //mb_pred |
5160 if(IS_INTRA(mb_type)){ | 5183 if(IS_INTRA(mb_type)){ |
5184 int pred_mode; | |
5161 // init_top_left_availability(h); | 5185 // init_top_left_availability(h); |
5162 if(IS_INTRA4x4(mb_type)){ | 5186 if(IS_INTRA4x4(mb_type)){ |
5163 int i; | 5187 int i; |
5164 int di = 1; | 5188 int di = 1; |
5165 if(dct8x8_allowed && get_bits1(&s->gb)){ | 5189 if(dct8x8_allowed && get_bits1(&s->gb)){ |
5187 }else{ | 5211 }else{ |
5188 h->intra16x16_pred_mode= check_intra_pred_mode(h, h->intra16x16_pred_mode); | 5212 h->intra16x16_pred_mode= check_intra_pred_mode(h, h->intra16x16_pred_mode); |
5189 if(h->intra16x16_pred_mode < 0) | 5213 if(h->intra16x16_pred_mode < 0) |
5190 return -1; | 5214 return -1; |
5191 } | 5215 } |
5192 h->chroma_pred_mode= get_ue_golomb(&s->gb); | 5216 |
5193 | 5217 pred_mode= check_intra_pred_mode(h, get_ue_golomb(&s->gb)); |
5194 h->chroma_pred_mode= check_intra_pred_mode(h, h->chroma_pred_mode); | 5218 if(pred_mode < 0) |
5195 if(h->chroma_pred_mode < 0) | |
5196 return -1; | 5219 return -1; |
5220 h->chroma_pred_mode= pred_mode; | |
5197 }else if(partition_count==4){ | 5221 }else if(partition_count==4){ |
5198 int i, j, sub_partition_count[4], list, ref[2][4]; | 5222 int i, j, sub_partition_count[4], list, ref[2][4]; |
5199 | 5223 |
5200 if(h->slice_type == B_TYPE){ | 5224 if(h->slice_type == B_TYPE){ |
5201 for(i=0; i<4; i++){ | 5225 for(i=0; i<4; i++){ |
5202 h->sub_mb_type[i]= get_ue_golomb(&s->gb); | 5226 h->sub_mb_type[i]= get_ue_golomb(&s->gb); |
5203 if(h->sub_mb_type[i] >=13){ | 5227 if(h->sub_mb_type[i] >=13){ |
5204 av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); | 5228 av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); |
5205 return -1; | 5229 return -1; |
5206 } | 5230 } |
5207 sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; | 5231 sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; |
5208 h->sub_mb_type[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].type; | 5232 h->sub_mb_type[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].type; |
5209 } | 5233 } |
5218 }else{ | 5242 }else{ |
5219 assert(h->slice_type == P_TYPE || h->slice_type == SP_TYPE); //FIXME SP correct ? | 5243 assert(h->slice_type == P_TYPE || h->slice_type == SP_TYPE); //FIXME SP correct ? |
5220 for(i=0; i<4; i++){ | 5244 for(i=0; i<4; i++){ |
5221 h->sub_mb_type[i]= get_ue_golomb(&s->gb); | 5245 h->sub_mb_type[i]= get_ue_golomb(&s->gb); |
5222 if(h->sub_mb_type[i] >=4){ | 5246 if(h->sub_mb_type[i] >=4){ |
5223 av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); | 5247 av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); |
5224 return -1; | 5248 return -1; |
5225 } | 5249 } |
5226 sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; | 5250 sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; |
5227 h->sub_mb_type[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].type; | 5251 h->sub_mb_type[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].type; |
5228 } | 5252 } |
5232 int ref_count= IS_REF0(mb_type) ? 1 : h->ref_count[list]; | 5256 int ref_count= IS_REF0(mb_type) ? 1 : h->ref_count[list]; |
5233 if(ref_count == 0) continue; | 5257 if(ref_count == 0) continue; |
5234 for(i=0; i<4; i++){ | 5258 for(i=0; i<4; i++){ |
5235 if(IS_DIRECT(h->sub_mb_type[i])) continue; | 5259 if(IS_DIRECT(h->sub_mb_type[i])) continue; |
5236 if(IS_DIR(h->sub_mb_type[i], 0, list)){ | 5260 if(IS_DIR(h->sub_mb_type[i], 0, list)){ |
5237 ref[list][i] = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip? | 5261 unsigned int tmp = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip? |
5262 if(tmp>=ref_count){ | |
5263 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", tmp); | |
5264 return -1; | |
5265 } | |
5266 ref[list][i]= tmp; | |
5238 }else{ | 5267 }else{ |
5239 //FIXME | 5268 //FIXME |
5240 ref[list][i] = -1; | 5269 ref[list][i] = -1; |
5241 } | 5270 } |
5242 } | 5271 } |
5301 //FIXME we should set ref_idx_l? to 0 if we use that later ... | 5330 //FIXME we should set ref_idx_l? to 0 if we use that later ... |
5302 if(IS_16X16(mb_type)){ | 5331 if(IS_16X16(mb_type)){ |
5303 for(list=0; list<2; list++){ | 5332 for(list=0; list<2; list++){ |
5304 if(h->ref_count[list]>0){ | 5333 if(h->ref_count[list]>0){ |
5305 if(IS_DIR(mb_type, 0, list)){ | 5334 if(IS_DIR(mb_type, 0, list)){ |
5306 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); | 5335 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); |
5336 if(val >= h->ref_count[list]){ | |
5337 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); | |
5338 return -1; | |
5339 } | |
5307 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1); | 5340 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1); |
5308 }else | 5341 }else |
5309 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1); | 5342 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1); |
5310 } | 5343 } |
5311 } | 5344 } |
5324 else if(IS_16X8(mb_type)){ | 5357 else if(IS_16X8(mb_type)){ |
5325 for(list=0; list<2; list++){ | 5358 for(list=0; list<2; list++){ |
5326 if(h->ref_count[list]>0){ | 5359 if(h->ref_count[list]>0){ |
5327 for(i=0; i<2; i++){ | 5360 for(i=0; i<2; i++){ |
5328 if(IS_DIR(mb_type, i, list)){ | 5361 if(IS_DIR(mb_type, i, list)){ |
5329 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); | 5362 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); |
5363 if(val >= h->ref_count[list]){ | |
5364 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); | |
5365 return -1; | |
5366 } | |
5330 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1); | 5367 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1); |
5331 }else | 5368 }else |
5332 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); | 5369 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); |
5333 } | 5370 } |
5334 } | 5371 } |
5350 assert(IS_8X16(mb_type)); | 5387 assert(IS_8X16(mb_type)); |
5351 for(list=0; list<2; list++){ | 5388 for(list=0; list<2; list++){ |
5352 if(h->ref_count[list]>0){ | 5389 if(h->ref_count[list]>0){ |
5353 for(i=0; i<2; i++){ | 5390 for(i=0; i<2; i++){ |
5354 if(IS_DIR(mb_type, i, list)){ //FIXME optimize | 5391 if(IS_DIR(mb_type, i, list)){ //FIXME optimize |
5355 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); | 5392 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]); |
5393 if(val >= h->ref_count[list]){ | |
5394 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val); | |
5395 return -1; | |
5396 } | |
5356 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1); | 5397 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1); |
5357 }else | 5398 }else |
5358 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); | 5399 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); |
5359 } | 5400 } |
5360 } | 5401 } |
5379 write_back_motion(h, mb_type); | 5420 write_back_motion(h, mb_type); |
5380 | 5421 |
5381 if(!IS_INTRA16x16(mb_type)){ | 5422 if(!IS_INTRA16x16(mb_type)){ |
5382 cbp= get_ue_golomb(&s->gb); | 5423 cbp= get_ue_golomb(&s->gb); |
5383 if(cbp > 47){ | 5424 if(cbp > 47){ |
5384 av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%d) at %d %d\n", cbp, s->mb_x, s->mb_y); | 5425 av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%u) at %d %d\n", cbp, s->mb_x, s->mb_y); |
5385 return -1; | 5426 return -1; |
5386 } | 5427 } |
5387 | 5428 |
5388 if(IS_INTRA4x4(mb_type)) | 5429 if(IS_INTRA4x4(mb_type)) |
5389 cbp= golomb_to_intra4x4_cbp[cbp]; | 5430 cbp= golomb_to_intra4x4_cbp[cbp]; |
5861 ref++; | 5902 ref++; |
5862 if( ctx < 4 ) | 5903 if( ctx < 4 ) |
5863 ctx = 4; | 5904 ctx = 4; |
5864 else | 5905 else |
5865 ctx = 5; | 5906 ctx = 5; |
5907 if(ref >= 32 /*h->ref_list[list]*/){ | |
5908 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n"); | |
5909 return 0; //FIXME we should return -1 and check the return everywhere | |
5910 } | |
5866 } | 5911 } |
5867 return ref; | 5912 return ref; |
5868 } | 5913 } |
5869 | 5914 |
5870 static int decode_cabac_mb_mvd( H264Context *h, int list, int n, int l ) { | 5915 static int decode_cabac_mb_mvd( H264Context *h, int list, int n, int l ) { |
5894 if( mvd >= 9 ) { | 5939 if( mvd >= 9 ) { |
5895 int k = 3; | 5940 int k = 3; |
5896 while( get_cabac_bypass( &h->cabac ) ) { | 5941 while( get_cabac_bypass( &h->cabac ) ) { |
5897 mvd += 1 << k; | 5942 mvd += 1 << k; |
5898 k++; | 5943 k++; |
5944 if(k>24){ | |
5945 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_mvd\n"); | |
5946 return INT_MIN; | |
5947 } | |
5899 } | 5948 } |
5900 while( k-- ) { | 5949 while( k-- ) { |
5901 if( get_cabac_bypass( &h->cabac ) ) | 5950 if( get_cabac_bypass( &h->cabac ) ) |
5902 mvd += 1 << k; | 5951 mvd += 1 << k; |
5903 } | 5952 } |
6283 } | 6332 } |
6284 | 6333 |
6285 fill_caches(h, mb_type, 0); | 6334 fill_caches(h, mb_type, 0); |
6286 | 6335 |
6287 if( IS_INTRA( mb_type ) ) { | 6336 if( IS_INTRA( mb_type ) ) { |
6288 int i; | 6337 int i, pred_mode; |
6289 if( IS_INTRA4x4( mb_type ) ) { | 6338 if( IS_INTRA4x4( mb_type ) ) { |
6290 if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) { | 6339 if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) { |
6291 mb_type |= MB_TYPE_8x8DCT; | 6340 mb_type |= MB_TYPE_8x8DCT; |
6292 for( i = 0; i < 16; i+=4 ) { | 6341 for( i = 0; i < 16; i+=4 ) { |
6293 int pred = pred_intra_mode( h, i ); | 6342 int pred = pred_intra_mode( h, i ); |
6307 } else { | 6356 } else { |
6308 h->intra16x16_pred_mode= check_intra_pred_mode( h, h->intra16x16_pred_mode ); | 6357 h->intra16x16_pred_mode= check_intra_pred_mode( h, h->intra16x16_pred_mode ); |
6309 if( h->intra16x16_pred_mode < 0 ) return -1; | 6358 if( h->intra16x16_pred_mode < 0 ) return -1; |
6310 } | 6359 } |
6311 h->chroma_pred_mode_table[mb_xy] = | 6360 h->chroma_pred_mode_table[mb_xy] = |
6312 h->chroma_pred_mode = decode_cabac_mb_chroma_pre_mode( h ); | 6361 pred_mode = decode_cabac_mb_chroma_pre_mode( h ); |
6313 | 6362 |
6314 h->chroma_pred_mode= check_intra_pred_mode( h, h->chroma_pred_mode ); | 6363 pred_mode= check_intra_pred_mode( h, pred_mode ); |
6315 if( h->chroma_pred_mode < 0 ) return -1; | 6364 if( pred_mode < 0 ) return -1; |
6365 h->chroma_pred_mode= pred_mode; | |
6316 } else if( partition_count == 4 ) { | 6366 } else if( partition_count == 4 ) { |
6317 int i, j, sub_partition_count[4], list, ref[2][4]; | 6367 int i, j, sub_partition_count[4], list, ref[2][4]; |
6318 | 6368 |
6319 if( h->slice_type == B_TYPE ) { | 6369 if( h->slice_type == B_TYPE ) { |
6320 for( i = 0; i < 4; i++ ) { | 6370 for( i = 0; i < 4; i++ ) { |
7559 get_bits(&s->gb, 5); /* time_offset_length */ | 7609 get_bits(&s->gb, 5); /* time_offset_length */ |
7560 } | 7610 } |
7561 | 7611 |
7562 static inline int decode_vui_parameters(H264Context *h, SPS *sps){ | 7612 static inline int decode_vui_parameters(H264Context *h, SPS *sps){ |
7563 MpegEncContext * const s = &h->s; | 7613 MpegEncContext * const s = &h->s; |
7564 int aspect_ratio_info_present_flag, aspect_ratio_idc; | 7614 int aspect_ratio_info_present_flag; |
7615 unsigned int aspect_ratio_idc; | |
7565 int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag; | 7616 int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag; |
7566 | 7617 |
7567 aspect_ratio_info_present_flag= get_bits1(&s->gb); | 7618 aspect_ratio_info_present_flag= get_bits1(&s->gb); |
7568 | 7619 |
7569 if( aspect_ratio_info_present_flag ) { | 7620 if( aspect_ratio_info_present_flag ) { |
7681 } | 7732 } |
7682 | 7733 |
7683 static inline int decode_seq_parameter_set(H264Context *h){ | 7734 static inline int decode_seq_parameter_set(H264Context *h){ |
7684 MpegEncContext * const s = &h->s; | 7735 MpegEncContext * const s = &h->s; |
7685 int profile_idc, level_idc; | 7736 int profile_idc, level_idc; |
7686 unsigned int sps_id; | 7737 unsigned int sps_id, tmp, mb_width, mb_height; |
7687 int i; | 7738 int i; |
7688 SPS *sps; | 7739 SPS *sps; |
7689 | 7740 |
7690 profile_idc= get_bits(&s->gb, 8); | 7741 profile_idc= get_bits(&s->gb, 8); |
7691 get_bits1(&s->gb); //constraint_set0_flag | 7742 get_bits1(&s->gb); //constraint_set0_flag |
7723 sps->log2_max_poc_lsb= get_ue_golomb(&s->gb) + 4; | 7774 sps->log2_max_poc_lsb= get_ue_golomb(&s->gb) + 4; |
7724 } else if(sps->poc_type == 1){//FIXME #define | 7775 } else if(sps->poc_type == 1){//FIXME #define |
7725 sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb); | 7776 sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb); |
7726 sps->offset_for_non_ref_pic= get_se_golomb(&s->gb); | 7777 sps->offset_for_non_ref_pic= get_se_golomb(&s->gb); |
7727 sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb); | 7778 sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb); |
7728 sps->poc_cycle_length= get_ue_golomb(&s->gb); | 7779 tmp= get_ue_golomb(&s->gb); |
7780 | |
7781 if(tmp >= sizeof(sps->offset_for_ref_frame) / sizeof(sps->offset_for_ref_frame[0])){ | |
7782 av_log(h->s.avctx, AV_LOG_ERROR, "poc_cycle_length overflow %u\n", tmp); | |
7783 return -1; | |
7784 } | |
7785 sps->poc_cycle_length= tmp; | |
7729 | 7786 |
7730 for(i=0; i<sps->poc_cycle_length; i++) | 7787 for(i=0; i<sps->poc_cycle_length; i++) |
7731 sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb); | 7788 sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb); |
7732 } | 7789 }else if(sps->poc_type != 2){ |
7733 if(sps->poc_type > 2){ | |
7734 av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type); | 7790 av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type); |
7735 return -1; | 7791 return -1; |
7736 } | 7792 } |
7737 | 7793 |
7738 sps->ref_frame_count= get_ue_golomb(&s->gb); | 7794 tmp= get_ue_golomb(&s->gb); |
7739 if(sps->ref_frame_count > MAX_PICTURE_COUNT-2){ | 7795 if(tmp > MAX_PICTURE_COUNT-2){ |
7740 av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n"); | 7796 av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n"); |
7741 } | 7797 } |
7798 sps->ref_frame_count= tmp; | |
7742 sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb); | 7799 sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb); |
7743 sps->mb_width= get_ue_golomb(&s->gb) + 1; | 7800 mb_width= get_ue_golomb(&s->gb) + 1; |
7744 sps->mb_height= get_ue_golomb(&s->gb) + 1; | 7801 mb_height= get_ue_golomb(&s->gb) + 1; |
7745 if((unsigned)sps->mb_width >= INT_MAX/16 || (unsigned)sps->mb_height >= INT_MAX/16 || | 7802 if(mb_width >= INT_MAX/16 || mb_height >= INT_MAX/16 || |
7746 avcodec_check_dimensions(NULL, 16*sps->mb_width, 16*sps->mb_height)) | 7803 avcodec_check_dimensions(NULL, 16*mb_width, 16*mb_height)){ |
7804 av_log(h->s.avctx, AV_LOG_ERROR, "mb_width/height overflow\n"); | |
7747 return -1; | 7805 return -1; |
7806 } | |
7807 sps->mb_width = mb_width; | |
7808 sps->mb_height= mb_height; | |
7748 | 7809 |
7749 sps->frame_mbs_only_flag= get_bits1(&s->gb); | 7810 sps->frame_mbs_only_flag= get_bits1(&s->gb); |
7750 if(!sps->frame_mbs_only_flag) | 7811 if(!sps->frame_mbs_only_flag) |
7751 sps->mb_aff= get_bits1(&s->gb); | 7812 sps->mb_aff= get_bits1(&s->gb); |
7752 else | 7813 else |
7797 return 0; | 7858 return 0; |
7798 } | 7859 } |
7799 | 7860 |
7800 static inline int decode_picture_parameter_set(H264Context *h, int bit_length){ | 7861 static inline int decode_picture_parameter_set(H264Context *h, int bit_length){ |
7801 MpegEncContext * const s = &h->s; | 7862 MpegEncContext * const s = &h->s; |
7802 unsigned int pps_id= get_ue_golomb(&s->gb); | 7863 unsigned int tmp, pps_id= get_ue_golomb(&s->gb); |
7803 PPS *pps; | 7864 PPS *pps; |
7804 | 7865 |
7805 if(pps_id>=MAX_PPS_COUNT){ | 7866 if(pps_id>=MAX_PPS_COUNT){ |
7806 av_log(h->s.avctx, AV_LOG_ERROR, "pps_id out of range\n"); | 7867 av_log(h->s.avctx, AV_LOG_ERROR, "pps_id out of range\n"); |
7807 return -1; | 7868 return -1; |
7808 } | 7869 } |
7809 pps = &h->pps_buffer[pps_id]; | 7870 pps = &h->pps_buffer[pps_id]; |
7810 | 7871 |
7811 pps->sps_id= get_ue_golomb(&s->gb); | 7872 tmp= get_ue_golomb(&s->gb); |
7873 if(tmp>=MAX_SPS_COUNT){ | |
7874 av_log(h->s.avctx, AV_LOG_ERROR, "sps_id out of range\n"); | |
7875 return -1; | |
7876 } | |
7877 pps->sps_id= tmp; | |
7878 | |
7812 pps->cabac= get_bits1(&s->gb); | 7879 pps->cabac= get_bits1(&s->gb); |
7813 pps->pic_order_present= get_bits1(&s->gb); | 7880 pps->pic_order_present= get_bits1(&s->gb); |
7814 pps->slice_group_count= get_ue_golomb(&s->gb) + 1; | 7881 pps->slice_group_count= get_ue_golomb(&s->gb) + 1; |
7815 if(pps->slice_group_count > 1 ){ | 7882 if(pps->slice_group_count > 1 ){ |
7816 pps->mb_slice_group_map_type= get_ue_golomb(&s->gb); | 7883 pps->mb_slice_group_map_type= get_ue_golomb(&s->gb); |
7851 } | 7918 } |
7852 pps->ref_count[0]= get_ue_golomb(&s->gb) + 1; | 7919 pps->ref_count[0]= get_ue_golomb(&s->gb) + 1; |
7853 pps->ref_count[1]= get_ue_golomb(&s->gb) + 1; | 7920 pps->ref_count[1]= get_ue_golomb(&s->gb) + 1; |
7854 if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){ | 7921 if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){ |
7855 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n"); | 7922 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n"); |
7923 pps->ref_count[0]= pps->ref_count[1]= 1; | |
7856 return -1; | 7924 return -1; |
7857 } | 7925 } |
7858 | 7926 |
7859 pps->weighted_pred= get_bits1(&s->gb); | 7927 pps->weighted_pred= get_bits1(&s->gb); |
7860 pps->weighted_bipred_idc= get_bits(&s->gb, 2); | 7928 pps->weighted_bipred_idc= get_bits(&s->gb, 2); |