comparison h264.c @ 4365:9cebff821565 libavcodec

checking bitstream values and other related changes some of these might have been exploitable
author michael
date Sat, 20 Jan 2007 13:50:00 +0000
parents 05e932ddaaa9
children 3451831c6962
comparison
equal deleted inserted replaced
4364:05e932ddaaa9 4365:9cebff821565
120 unsigned int sps_id; 120 unsigned int sps_id;
121 int cabac; ///< entropy_coding_mode_flag 121 int cabac; ///< entropy_coding_mode_flag
122 int pic_order_present; ///< pic_order_present_flag 122 int pic_order_present; ///< pic_order_present_flag
123 int slice_group_count; ///< num_slice_groups_minus1 + 1 123 int slice_group_count; ///< num_slice_groups_minus1 + 1
124 int mb_slice_group_map_type; 124 int mb_slice_group_map_type;
125 int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1 125 unsigned int ref_count[2]; ///< num_ref_idx_l0/1_active_minus1 + 1
126 int weighted_pred; ///< weighted_pred_flag 126 int weighted_pred; ///< weighted_pred_flag
127 int weighted_bipred_idc; 127 int weighted_bipred_idc;
128 int init_qp; ///< pic_init_qp_minus26 + 26 128 int init_qp; ///< pic_init_qp_minus26 + 26
129 int init_qs; ///< pic_init_qs_minus26 + 26 129 int init_qs; ///< pic_init_qs_minus26 + 26
130 int chroma_qp_index_offset; 130 int chroma_qp_index_offset;
272 //interlacing specific flags 272 //interlacing specific flags
273 int mb_aff_frame; 273 int mb_aff_frame;
274 int mb_field_decoding_flag; 274 int mb_field_decoding_flag;
275 int mb_mbaff; ///< mb_aff_frame && mb_field_decoding_flag 275 int mb_mbaff; ///< mb_aff_frame && mb_field_decoding_flag
276 276
277 int sub_mb_type[4]; 277 unsigned int sub_mb_type[4];
278 278
279 //POC stuff 279 //POC stuff
280 int poc_lsb; 280 int poc_lsb;
281 int poc_msb; 281 int poc_msb;
282 int delta_poc_bottom; 282 int delta_poc_bottom;
323 int map_col_to_list0_field[2][32]; 323 int map_col_to_list0_field[2][32];
324 324
325 /** 325 /**
326 * num_ref_idx_l0/1_active_minus1 + 1 326 * num_ref_idx_l0/1_active_minus1 + 1
327 */ 327 */
328 int ref_count[2]; ///< counts frames or fields, depending on current mb mode 328 unsigned int ref_count[2]; ///< counts frames or fields, depending on current mb mode
329 Picture *short_ref[32]; 329 Picture *short_ref[32];
330 Picture *long_ref[32]; 330 Picture *long_ref[32];
331 Picture default_ref_list[2][32]; 331 Picture default_ref_list[2][32];
332 Picture ref_list[2][48]; ///< 0..15: frame refs, 16..47: mbaff field refs 332 Picture ref_list[2][48]; ///< 0..15: frame refs, 16..47: mbaff field refs
333 Picture *delayed_pic[16]; //FIXME size? 333 Picture *delayed_pic[16]; //FIXME size?
347 GetBitContext inter_gb; 347 GetBitContext inter_gb;
348 GetBitContext *intra_gb_ptr; 348 GetBitContext *intra_gb_ptr;
349 GetBitContext *inter_gb_ptr; 349 GetBitContext *inter_gb_ptr;
350 350
351 DECLARE_ALIGNED_8(DCTELEM, mb[16*24]); 351 DECLARE_ALIGNED_8(DCTELEM, mb[16*24]);
352 DCTELEM mb_padding[256]; ///< as mb is addressed by scantable[i] and scantable is uint8_t we can either check that i is not to large or ensure that there is some unused stuff after mb
352 353
353 /** 354 /**
354 * Cabac 355 * Cabac
355 */ 356 */
356 CABACContext cabac; 357 CABACContext cabac;
1396 const int16_t (*l1mv0)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[0][b4_xy]; 1397 const int16_t (*l1mv0)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[0][b4_xy];
1397 const int16_t (*l1mv1)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[1][b4_xy]; 1398 const int16_t (*l1mv1)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[1][b4_xy];
1398 const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy]; 1399 const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy];
1399 const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy]; 1400 const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy];
1400 const int is_b8x8 = IS_8X8(*mb_type); 1401 const int is_b8x8 = IS_8X8(*mb_type);
1401 int sub_mb_type; 1402 unsigned int sub_mb_type;
1402 int i8, i4; 1403 int i8, i4;
1403 1404
1404 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM) 1405 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM)
1405 if(IS_8X8(mb_type_col) && !h->sps.direct_8x8_inference_flag){ 1406 if(IS_8X8(mb_type_col) && !h->sps.direct_8x8_inference_flag){
1406 /* FIXME save sub mb types from previous frames (or derive from MVs) 1407 /* FIXME save sub mb types from previous frames (or derive from MVs)
3914 3915
3915 if(get_bits1(&s->gb)){ 3916 if(get_bits1(&s->gb)){
3916 int pred= h->curr_pic_num; 3917 int pred= h->curr_pic_num;
3917 3918
3918 for(index=0; ; index++){ 3919 for(index=0; ; index++){
3919 int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb); 3920 unsigned int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb);
3920 int pic_id; 3921 unsigned int pic_id;
3921 int i; 3922 int i;
3922 Picture *ref = NULL; 3923 Picture *ref = NULL;
3923 3924
3924 if(reordering_of_pic_nums_idc==3) 3925 if(reordering_of_pic_nums_idc==3)
3925 break; 3926 break;
3929 return -1; 3930 return -1;
3930 } 3931 }
3931 3932
3932 if(reordering_of_pic_nums_idc<3){ 3933 if(reordering_of_pic_nums_idc<3){
3933 if(reordering_of_pic_nums_idc<2){ 3934 if(reordering_of_pic_nums_idc<2){
3934 const int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1; 3935 const unsigned int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1;
3935 3936
3936 if(abs_diff_pic_num >= h->max_pic_num){ 3937 if(abs_diff_pic_num >= h->max_pic_num){
3937 av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n"); 3938 av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n");
3938 return -1; 3939 return -1;
3939 } 3940 }
3951 } 3952 }
3952 if(i>=0) 3953 if(i>=0)
3953 ref->pic_id= ref->frame_num; 3954 ref->pic_id= ref->frame_num;
3954 }else{ 3955 }else{
3955 pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx 3956 pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx
3957 if(pic_id>31){
3958 av_log(h->s.avctx, AV_LOG_ERROR, "long_term_pic_idx overflow\n");
3959 return -1;
3960 }
3956 ref = h->long_ref[pic_id]; 3961 ref = h->long_ref[pic_id];
3957 if(ref){ 3962 if(ref){
3958 ref->pic_id= pic_id; 3963 ref->pic_id= pic_id;
3959 assert(ref->reference == 3); 3964 assert(ref->reference == 3);
3960 assert(ref->long_ref); 3965 assert(ref->long_ref);
4356 av_log(s->avctx, AV_LOG_ERROR, "illegal short ref in memory management control operation %d\n", mmco); 4361 av_log(s->avctx, AV_LOG_ERROR, "illegal short ref in memory management control operation %d\n", mmco);
4357 return -1; 4362 return -1;
4358 }*/ 4363 }*/
4359 } 4364 }
4360 if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){ 4365 if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){
4361 h->mmco[i].long_index= get_ue_golomb(&s->gb); 4366 unsigned int long_index= get_ue_golomb(&s->gb);
4362 if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ h->mmco[i].long_index >= 16){ 4367 if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ long_index >= 16){
4363 av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode); 4368 av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode);
4364 return -1; 4369 return -1;
4365 } 4370 }
4366 } 4371 h->mmco[i].long_index= long_index;
4367 4372 }
4368 if(opcode > MMCO_LONG){ 4373
4374 if(opcode > (unsigned)MMCO_LONG){
4369 av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode); 4375 av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode);
4370 return -1; 4376 return -1;
4371 } 4377 }
4372 if(opcode == MMCO_END) 4378 if(opcode == MMCO_END)
4373 break; 4379 break;
4481 * decodes a slice header. 4487 * decodes a slice header.
4482 * this will allso call MPV_common_init() and frame_start() as needed 4488 * this will allso call MPV_common_init() and frame_start() as needed
4483 */ 4489 */
4484 static int decode_slice_header(H264Context *h){ 4490 static int decode_slice_header(H264Context *h){
4485 MpegEncContext * const s = &h->s; 4491 MpegEncContext * const s = &h->s;
4486 int first_mb_in_slice; 4492 unsigned int first_mb_in_slice;
4487 unsigned int pps_id; 4493 unsigned int pps_id;
4488 int num_ref_idx_active_override_flag; 4494 int num_ref_idx_active_override_flag;
4489 static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE}; 4495 static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE};
4490 int slice_type; 4496 unsigned int slice_type, tmp;
4491 int default_ref_list_done = 0; 4497 int default_ref_list_done = 0;
4492 4498
4493 s->current_picture.reference= h->nal_ref_idc != 0; 4499 s->current_picture.reference= h->nal_ref_idc != 0;
4494 s->dropable= h->nal_ref_idc == 0; 4500 s->dropable= h->nal_ref_idc == 0;
4495 4501
4638 } else { 4644 } else {
4639 s->picture_structure= PICT_FRAME; 4645 s->picture_structure= PICT_FRAME;
4640 h->mb_aff_frame = h->sps.mb_aff; 4646 h->mb_aff_frame = h->sps.mb_aff;
4641 } 4647 }
4642 } 4648 }
4643 4649 assert(s->mb_num == s->mb_width * s->mb_height);
4650 if(first_mb_in_slice << h->mb_aff_frame >= s->mb_num ||
4651 first_mb_in_slice >= s->mb_num){
4652 av_log(h->s.avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n");
4653 return -1;
4654 }
4644 s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width; 4655 s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width;
4645 s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame; 4656 s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame;
4646 if(s->mb_y >= s->mb_height){ 4657 assert(s->mb_y < s->mb_height);
4647 return -1;
4648 }
4649 4658
4650 if(s->picture_structure==PICT_FRAME){ 4659 if(s->picture_structure==PICT_FRAME){
4651 h->curr_pic_num= h->frame_num; 4660 h->curr_pic_num= h->frame_num;
4652 h->max_pic_num= 1<< h->sps.log2_max_frame_num; 4661 h->max_pic_num= 1<< h->sps.log2_max_frame_num;
4653 }else{ 4662 }else{
4697 if(h->slice_type==B_TYPE) 4706 if(h->slice_type==B_TYPE)
4698 h->ref_count[1]= get_ue_golomb(&s->gb) + 1; 4707 h->ref_count[1]= get_ue_golomb(&s->gb) + 1;
4699 4708
4700 if(h->ref_count[0] > 32 || h->ref_count[1] > 32){ 4709 if(h->ref_count[0] > 32 || h->ref_count[1] > 32){
4701 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); 4710 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n");
4711 h->ref_count[0]= h->ref_count[1]= 1;
4702 return -1; 4712 return -1;
4703 } 4713 }
4704 } 4714 }
4705 } 4715 }
4706 4716
4723 decode_ref_pic_marking(h); 4733 decode_ref_pic_marking(h);
4724 4734
4725 if(FRAME_MBAFF) 4735 if(FRAME_MBAFF)
4726 fill_mbaff_ref_list(h); 4736 fill_mbaff_ref_list(h);
4727 4737
4728 if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ) 4738 if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ){
4729 h->cabac_init_idc = get_ue_golomb(&s->gb); 4739 tmp = get_ue_golomb(&s->gb);
4740 if(tmp > 2){
4741 av_log(s->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n");
4742 return -1;
4743 }
4744 h->cabac_init_idc= tmp;
4745 }
4730 4746
4731 h->last_qscale_diff = 0; 4747 h->last_qscale_diff = 0;
4732 s->qscale = h->pps.init_qp + get_se_golomb(&s->gb); 4748 tmp = h->pps.init_qp + get_se_golomb(&s->gb);
4733 if(s->qscale<0 || s->qscale>51){ 4749 if(tmp>51){
4734 av_log(s->avctx, AV_LOG_ERROR, "QP %d out of range\n", s->qscale); 4750 av_log(s->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp);
4735 return -1; 4751 return -1;
4736 } 4752 }
4753 s->qscale= tmp;
4737 h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale); 4754 h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale);
4738 //FIXME qscale / qp ... stuff 4755 //FIXME qscale / qp ... stuff
4739 if(h->slice_type == SP_TYPE){ 4756 if(h->slice_type == SP_TYPE){
4740 get_bits1(&s->gb); /* sp_for_switch_flag */ 4757 get_bits1(&s->gb); /* sp_for_switch_flag */
4741 } 4758 }
4745 4762
4746 h->deblocking_filter = 1; 4763 h->deblocking_filter = 1;
4747 h->slice_alpha_c0_offset = 0; 4764 h->slice_alpha_c0_offset = 0;
4748 h->slice_beta_offset = 0; 4765 h->slice_beta_offset = 0;
4749 if( h->pps.deblocking_filter_parameters_present ) { 4766 if( h->pps.deblocking_filter_parameters_present ) {
4750 h->deblocking_filter= get_ue_golomb(&s->gb); 4767 tmp= get_ue_golomb(&s->gb);
4768 if(tmp > 2){
4769 av_log(s->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp);
4770 return -1;
4771 }
4772 h->deblocking_filter= tmp;
4751 if(h->deblocking_filter < 2) 4773 if(h->deblocking_filter < 2)
4752 h->deblocking_filter^= 1; // 1<->0 4774 h->deblocking_filter^= 1; // 1<->0
4753 4775
4754 if( h->deblocking_filter ) { 4776 if( h->deblocking_filter ) {
4755 h->slice_alpha_c0_offset = get_se_golomb(&s->gb) << 1; 4777 h->slice_alpha_c0_offset = get_se_golomb(&s->gb) << 1;
5042 * @returns 0 if ok, AC_ERROR / DC_ERROR / MV_ERROR if an error is noticed 5064 * @returns 0 if ok, AC_ERROR / DC_ERROR / MV_ERROR if an error is noticed
5043 */ 5065 */
5044 static int decode_mb_cavlc(H264Context *h){ 5066 static int decode_mb_cavlc(H264Context *h){
5045 MpegEncContext * const s = &h->s; 5067 MpegEncContext * const s = &h->s;
5046 const int mb_xy= s->mb_x + s->mb_y*s->mb_stride; 5068 const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
5047 int mb_type, partition_count, cbp; 5069 int partition_count;
5070 unsigned int mb_type, cbp;
5048 int dct8x8_allowed= h->pps.transform_8x8_mode; 5071 int dct8x8_allowed= h->pps.transform_8x8_mode;
5049 5072
5050 s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong? 5073 s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong?
5051 5074
5052 tprintf("pic:%d mb:%d/%d\n", h->frame_num, s->mb_x, s->mb_y); 5075 tprintf("pic:%d mb:%d/%d\n", h->frame_num, s->mb_x, s->mb_y);
5156 5179
5157 fill_caches(h, mb_type, 0); 5180 fill_caches(h, mb_type, 0);
5158 5181
5159 //mb_pred 5182 //mb_pred
5160 if(IS_INTRA(mb_type)){ 5183 if(IS_INTRA(mb_type)){
5184 int pred_mode;
5161 // init_top_left_availability(h); 5185 // init_top_left_availability(h);
5162 if(IS_INTRA4x4(mb_type)){ 5186 if(IS_INTRA4x4(mb_type)){
5163 int i; 5187 int i;
5164 int di = 1; 5188 int di = 1;
5165 if(dct8x8_allowed && get_bits1(&s->gb)){ 5189 if(dct8x8_allowed && get_bits1(&s->gb)){
5187 }else{ 5211 }else{
5188 h->intra16x16_pred_mode= check_intra_pred_mode(h, h->intra16x16_pred_mode); 5212 h->intra16x16_pred_mode= check_intra_pred_mode(h, h->intra16x16_pred_mode);
5189 if(h->intra16x16_pred_mode < 0) 5213 if(h->intra16x16_pred_mode < 0)
5190 return -1; 5214 return -1;
5191 } 5215 }
5192 h->chroma_pred_mode= get_ue_golomb(&s->gb); 5216
5193 5217 pred_mode= check_intra_pred_mode(h, get_ue_golomb(&s->gb));
5194 h->chroma_pred_mode= check_intra_pred_mode(h, h->chroma_pred_mode); 5218 if(pred_mode < 0)
5195 if(h->chroma_pred_mode < 0)
5196 return -1; 5219 return -1;
5220 h->chroma_pred_mode= pred_mode;
5197 }else if(partition_count==4){ 5221 }else if(partition_count==4){
5198 int i, j, sub_partition_count[4], list, ref[2][4]; 5222 int i, j, sub_partition_count[4], list, ref[2][4];
5199 5223
5200 if(h->slice_type == B_TYPE){ 5224 if(h->slice_type == B_TYPE){
5201 for(i=0; i<4; i++){ 5225 for(i=0; i<4; i++){
5202 h->sub_mb_type[i]= get_ue_golomb(&s->gb); 5226 h->sub_mb_type[i]= get_ue_golomb(&s->gb);
5203 if(h->sub_mb_type[i] >=13){ 5227 if(h->sub_mb_type[i] >=13){
5204 av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); 5228 av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
5205 return -1; 5229 return -1;
5206 } 5230 }
5207 sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; 5231 sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count;
5208 h->sub_mb_type[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].type; 5232 h->sub_mb_type[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].type;
5209 } 5233 }
5218 }else{ 5242 }else{
5219 assert(h->slice_type == P_TYPE || h->slice_type == SP_TYPE); //FIXME SP correct ? 5243 assert(h->slice_type == P_TYPE || h->slice_type == SP_TYPE); //FIXME SP correct ?
5220 for(i=0; i<4; i++){ 5244 for(i=0; i<4; i++){
5221 h->sub_mb_type[i]= get_ue_golomb(&s->gb); 5245 h->sub_mb_type[i]= get_ue_golomb(&s->gb);
5222 if(h->sub_mb_type[i] >=4){ 5246 if(h->sub_mb_type[i] >=4){
5223 av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y); 5247 av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
5224 return -1; 5248 return -1;
5225 } 5249 }
5226 sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count; 5250 sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count;
5227 h->sub_mb_type[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].type; 5251 h->sub_mb_type[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].type;
5228 } 5252 }
5232 int ref_count= IS_REF0(mb_type) ? 1 : h->ref_count[list]; 5256 int ref_count= IS_REF0(mb_type) ? 1 : h->ref_count[list];
5233 if(ref_count == 0) continue; 5257 if(ref_count == 0) continue;
5234 for(i=0; i<4; i++){ 5258 for(i=0; i<4; i++){
5235 if(IS_DIRECT(h->sub_mb_type[i])) continue; 5259 if(IS_DIRECT(h->sub_mb_type[i])) continue;
5236 if(IS_DIR(h->sub_mb_type[i], 0, list)){ 5260 if(IS_DIR(h->sub_mb_type[i], 0, list)){
5237 ref[list][i] = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip? 5261 unsigned int tmp = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip?
5262 if(tmp>=ref_count){
5263 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", tmp);
5264 return -1;
5265 }
5266 ref[list][i]= tmp;
5238 }else{ 5267 }else{
5239 //FIXME 5268 //FIXME
5240 ref[list][i] = -1; 5269 ref[list][i] = -1;
5241 } 5270 }
5242 } 5271 }
5301 //FIXME we should set ref_idx_l? to 0 if we use that later ... 5330 //FIXME we should set ref_idx_l? to 0 if we use that later ...
5302 if(IS_16X16(mb_type)){ 5331 if(IS_16X16(mb_type)){
5303 for(list=0; list<2; list++){ 5332 for(list=0; list<2; list++){
5304 if(h->ref_count[list]>0){ 5333 if(h->ref_count[list]>0){
5305 if(IS_DIR(mb_type, 0, list)){ 5334 if(IS_DIR(mb_type, 0, list)){
5306 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); 5335 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
5336 if(val >= h->ref_count[list]){
5337 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
5338 return -1;
5339 }
5307 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1); 5340 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1);
5308 }else 5341 }else
5309 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1); 5342 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1);
5310 } 5343 }
5311 } 5344 }
5324 else if(IS_16X8(mb_type)){ 5357 else if(IS_16X8(mb_type)){
5325 for(list=0; list<2; list++){ 5358 for(list=0; list<2; list++){
5326 if(h->ref_count[list]>0){ 5359 if(h->ref_count[list]>0){
5327 for(i=0; i<2; i++){ 5360 for(i=0; i<2; i++){
5328 if(IS_DIR(mb_type, i, list)){ 5361 if(IS_DIR(mb_type, i, list)){
5329 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); 5362 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
5363 if(val >= h->ref_count[list]){
5364 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
5365 return -1;
5366 }
5330 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1); 5367 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1);
5331 }else 5368 }else
5332 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); 5369 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1);
5333 } 5370 }
5334 } 5371 }
5350 assert(IS_8X16(mb_type)); 5387 assert(IS_8X16(mb_type));
5351 for(list=0; list<2; list++){ 5388 for(list=0; list<2; list++){
5352 if(h->ref_count[list]>0){ 5389 if(h->ref_count[list]>0){
5353 for(i=0; i<2; i++){ 5390 for(i=0; i<2; i++){
5354 if(IS_DIR(mb_type, i, list)){ //FIXME optimize 5391 if(IS_DIR(mb_type, i, list)){ //FIXME optimize
5355 const int val= get_te0_golomb(&s->gb, h->ref_count[list]); 5392 unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
5393 if(val >= h->ref_count[list]){
5394 av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
5395 return -1;
5396 }
5356 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1); 5397 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1);
5357 }else 5398 }else
5358 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); 5399 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1);
5359 } 5400 }
5360 } 5401 }
5379 write_back_motion(h, mb_type); 5420 write_back_motion(h, mb_type);
5380 5421
5381 if(!IS_INTRA16x16(mb_type)){ 5422 if(!IS_INTRA16x16(mb_type)){
5382 cbp= get_ue_golomb(&s->gb); 5423 cbp= get_ue_golomb(&s->gb);
5383 if(cbp > 47){ 5424 if(cbp > 47){
5384 av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%d) at %d %d\n", cbp, s->mb_x, s->mb_y); 5425 av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%u) at %d %d\n", cbp, s->mb_x, s->mb_y);
5385 return -1; 5426 return -1;
5386 } 5427 }
5387 5428
5388 if(IS_INTRA4x4(mb_type)) 5429 if(IS_INTRA4x4(mb_type))
5389 cbp= golomb_to_intra4x4_cbp[cbp]; 5430 cbp= golomb_to_intra4x4_cbp[cbp];
5861 ref++; 5902 ref++;
5862 if( ctx < 4 ) 5903 if( ctx < 4 )
5863 ctx = 4; 5904 ctx = 4;
5864 else 5905 else
5865 ctx = 5; 5906 ctx = 5;
5907 if(ref >= 32 /*h->ref_list[list]*/){
5908 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n");
5909 return 0; //FIXME we should return -1 and check the return everywhere
5910 }
5866 } 5911 }
5867 return ref; 5912 return ref;
5868 } 5913 }
5869 5914
5870 static int decode_cabac_mb_mvd( H264Context *h, int list, int n, int l ) { 5915 static int decode_cabac_mb_mvd( H264Context *h, int list, int n, int l ) {
5894 if( mvd >= 9 ) { 5939 if( mvd >= 9 ) {
5895 int k = 3; 5940 int k = 3;
5896 while( get_cabac_bypass( &h->cabac ) ) { 5941 while( get_cabac_bypass( &h->cabac ) ) {
5897 mvd += 1 << k; 5942 mvd += 1 << k;
5898 k++; 5943 k++;
5944 if(k>24){
5945 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_mvd\n");
5946 return INT_MIN;
5947 }
5899 } 5948 }
5900 while( k-- ) { 5949 while( k-- ) {
5901 if( get_cabac_bypass( &h->cabac ) ) 5950 if( get_cabac_bypass( &h->cabac ) )
5902 mvd += 1 << k; 5951 mvd += 1 << k;
5903 } 5952 }
6283 } 6332 }
6284 6333
6285 fill_caches(h, mb_type, 0); 6334 fill_caches(h, mb_type, 0);
6286 6335
6287 if( IS_INTRA( mb_type ) ) { 6336 if( IS_INTRA( mb_type ) ) {
6288 int i; 6337 int i, pred_mode;
6289 if( IS_INTRA4x4( mb_type ) ) { 6338 if( IS_INTRA4x4( mb_type ) ) {
6290 if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) { 6339 if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) {
6291 mb_type |= MB_TYPE_8x8DCT; 6340 mb_type |= MB_TYPE_8x8DCT;
6292 for( i = 0; i < 16; i+=4 ) { 6341 for( i = 0; i < 16; i+=4 ) {
6293 int pred = pred_intra_mode( h, i ); 6342 int pred = pred_intra_mode( h, i );
6307 } else { 6356 } else {
6308 h->intra16x16_pred_mode= check_intra_pred_mode( h, h->intra16x16_pred_mode ); 6357 h->intra16x16_pred_mode= check_intra_pred_mode( h, h->intra16x16_pred_mode );
6309 if( h->intra16x16_pred_mode < 0 ) return -1; 6358 if( h->intra16x16_pred_mode < 0 ) return -1;
6310 } 6359 }
6311 h->chroma_pred_mode_table[mb_xy] = 6360 h->chroma_pred_mode_table[mb_xy] =
6312 h->chroma_pred_mode = decode_cabac_mb_chroma_pre_mode( h ); 6361 pred_mode = decode_cabac_mb_chroma_pre_mode( h );
6313 6362
6314 h->chroma_pred_mode= check_intra_pred_mode( h, h->chroma_pred_mode ); 6363 pred_mode= check_intra_pred_mode( h, pred_mode );
6315 if( h->chroma_pred_mode < 0 ) return -1; 6364 if( pred_mode < 0 ) return -1;
6365 h->chroma_pred_mode= pred_mode;
6316 } else if( partition_count == 4 ) { 6366 } else if( partition_count == 4 ) {
6317 int i, j, sub_partition_count[4], list, ref[2][4]; 6367 int i, j, sub_partition_count[4], list, ref[2][4];
6318 6368
6319 if( h->slice_type == B_TYPE ) { 6369 if( h->slice_type == B_TYPE ) {
6320 for( i = 0; i < 4; i++ ) { 6370 for( i = 0; i < 4; i++ ) {
7559 get_bits(&s->gb, 5); /* time_offset_length */ 7609 get_bits(&s->gb, 5); /* time_offset_length */
7560 } 7610 }
7561 7611
7562 static inline int decode_vui_parameters(H264Context *h, SPS *sps){ 7612 static inline int decode_vui_parameters(H264Context *h, SPS *sps){
7563 MpegEncContext * const s = &h->s; 7613 MpegEncContext * const s = &h->s;
7564 int aspect_ratio_info_present_flag, aspect_ratio_idc; 7614 int aspect_ratio_info_present_flag;
7615 unsigned int aspect_ratio_idc;
7565 int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag; 7616 int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag;
7566 7617
7567 aspect_ratio_info_present_flag= get_bits1(&s->gb); 7618 aspect_ratio_info_present_flag= get_bits1(&s->gb);
7568 7619
7569 if( aspect_ratio_info_present_flag ) { 7620 if( aspect_ratio_info_present_flag ) {
7681 } 7732 }
7682 7733
7683 static inline int decode_seq_parameter_set(H264Context *h){ 7734 static inline int decode_seq_parameter_set(H264Context *h){
7684 MpegEncContext * const s = &h->s; 7735 MpegEncContext * const s = &h->s;
7685 int profile_idc, level_idc; 7736 int profile_idc, level_idc;
7686 unsigned int sps_id; 7737 unsigned int sps_id, tmp, mb_width, mb_height;
7687 int i; 7738 int i;
7688 SPS *sps; 7739 SPS *sps;
7689 7740
7690 profile_idc= get_bits(&s->gb, 8); 7741 profile_idc= get_bits(&s->gb, 8);
7691 get_bits1(&s->gb); //constraint_set0_flag 7742 get_bits1(&s->gb); //constraint_set0_flag
7723 sps->log2_max_poc_lsb= get_ue_golomb(&s->gb) + 4; 7774 sps->log2_max_poc_lsb= get_ue_golomb(&s->gb) + 4;
7724 } else if(sps->poc_type == 1){//FIXME #define 7775 } else if(sps->poc_type == 1){//FIXME #define
7725 sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb); 7776 sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb);
7726 sps->offset_for_non_ref_pic= get_se_golomb(&s->gb); 7777 sps->offset_for_non_ref_pic= get_se_golomb(&s->gb);
7727 sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb); 7778 sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb);
7728 sps->poc_cycle_length= get_ue_golomb(&s->gb); 7779 tmp= get_ue_golomb(&s->gb);
7780
7781 if(tmp >= sizeof(sps->offset_for_ref_frame) / sizeof(sps->offset_for_ref_frame[0])){
7782 av_log(h->s.avctx, AV_LOG_ERROR, "poc_cycle_length overflow %u\n", tmp);
7783 return -1;
7784 }
7785 sps->poc_cycle_length= tmp;
7729 7786
7730 for(i=0; i<sps->poc_cycle_length; i++) 7787 for(i=0; i<sps->poc_cycle_length; i++)
7731 sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb); 7788 sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb);
7732 } 7789 }else if(sps->poc_type != 2){
7733 if(sps->poc_type > 2){
7734 av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type); 7790 av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type);
7735 return -1; 7791 return -1;
7736 } 7792 }
7737 7793
7738 sps->ref_frame_count= get_ue_golomb(&s->gb); 7794 tmp= get_ue_golomb(&s->gb);
7739 if(sps->ref_frame_count > MAX_PICTURE_COUNT-2){ 7795 if(tmp > MAX_PICTURE_COUNT-2){
7740 av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n"); 7796 av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n");
7741 } 7797 }
7798 sps->ref_frame_count= tmp;
7742 sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb); 7799 sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb);
7743 sps->mb_width= get_ue_golomb(&s->gb) + 1; 7800 mb_width= get_ue_golomb(&s->gb) + 1;
7744 sps->mb_height= get_ue_golomb(&s->gb) + 1; 7801 mb_height= get_ue_golomb(&s->gb) + 1;
7745 if((unsigned)sps->mb_width >= INT_MAX/16 || (unsigned)sps->mb_height >= INT_MAX/16 || 7802 if(mb_width >= INT_MAX/16 || mb_height >= INT_MAX/16 ||
7746 avcodec_check_dimensions(NULL, 16*sps->mb_width, 16*sps->mb_height)) 7803 avcodec_check_dimensions(NULL, 16*mb_width, 16*mb_height)){
7804 av_log(h->s.avctx, AV_LOG_ERROR, "mb_width/height overflow\n");
7747 return -1; 7805 return -1;
7806 }
7807 sps->mb_width = mb_width;
7808 sps->mb_height= mb_height;
7748 7809
7749 sps->frame_mbs_only_flag= get_bits1(&s->gb); 7810 sps->frame_mbs_only_flag= get_bits1(&s->gb);
7750 if(!sps->frame_mbs_only_flag) 7811 if(!sps->frame_mbs_only_flag)
7751 sps->mb_aff= get_bits1(&s->gb); 7812 sps->mb_aff= get_bits1(&s->gb);
7752 else 7813 else
7797 return 0; 7858 return 0;
7798 } 7859 }
7799 7860
7800 static inline int decode_picture_parameter_set(H264Context *h, int bit_length){ 7861 static inline int decode_picture_parameter_set(H264Context *h, int bit_length){
7801 MpegEncContext * const s = &h->s; 7862 MpegEncContext * const s = &h->s;
7802 unsigned int pps_id= get_ue_golomb(&s->gb); 7863 unsigned int tmp, pps_id= get_ue_golomb(&s->gb);
7803 PPS *pps; 7864 PPS *pps;
7804 7865
7805 if(pps_id>=MAX_PPS_COUNT){ 7866 if(pps_id>=MAX_PPS_COUNT){
7806 av_log(h->s.avctx, AV_LOG_ERROR, "pps_id out of range\n"); 7867 av_log(h->s.avctx, AV_LOG_ERROR, "pps_id out of range\n");
7807 return -1; 7868 return -1;
7808 } 7869 }
7809 pps = &h->pps_buffer[pps_id]; 7870 pps = &h->pps_buffer[pps_id];
7810 7871
7811 pps->sps_id= get_ue_golomb(&s->gb); 7872 tmp= get_ue_golomb(&s->gb);
7873 if(tmp>=MAX_SPS_COUNT){
7874 av_log(h->s.avctx, AV_LOG_ERROR, "sps_id out of range\n");
7875 return -1;
7876 }
7877 pps->sps_id= tmp;
7878
7812 pps->cabac= get_bits1(&s->gb); 7879 pps->cabac= get_bits1(&s->gb);
7813 pps->pic_order_present= get_bits1(&s->gb); 7880 pps->pic_order_present= get_bits1(&s->gb);
7814 pps->slice_group_count= get_ue_golomb(&s->gb) + 1; 7881 pps->slice_group_count= get_ue_golomb(&s->gb) + 1;
7815 if(pps->slice_group_count > 1 ){ 7882 if(pps->slice_group_count > 1 ){
7816 pps->mb_slice_group_map_type= get_ue_golomb(&s->gb); 7883 pps->mb_slice_group_map_type= get_ue_golomb(&s->gb);
7851 } 7918 }
7852 pps->ref_count[0]= get_ue_golomb(&s->gb) + 1; 7919 pps->ref_count[0]= get_ue_golomb(&s->gb) + 1;
7853 pps->ref_count[1]= get_ue_golomb(&s->gb) + 1; 7920 pps->ref_count[1]= get_ue_golomb(&s->gb) + 1;
7854 if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){ 7921 if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){
7855 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n"); 7922 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n");
7923 pps->ref_count[0]= pps->ref_count[1]= 1;
7856 return -1; 7924 return -1;
7857 } 7925 }
7858 7926
7859 pps->weighted_pred= get_bits1(&s->gb); 7927 pps->weighted_pred= get_bits1(&s->gb);
7860 pps->weighted_bipred_idc= get_bits(&s->gb, 2); 7928 pps->weighted_bipred_idc= get_bits(&s->gb, 2);