Mercurial > libavcodec.hg
comparison h264.c @ 8391:c1eb379daf90 libavcodec
Check ref values in CABAC H.264 for validity.
author | michael |
---|---|
date | Thu, 18 Dec 2008 23:31:10 +0000 |
parents | 31f8d52753e4 |
children | ee23da1a0c74 |
comparison
equal
deleted
inserted
replaced
8390:31f8d52753e4 | 8391:c1eb379daf90 |
---|---|
5030 if( ctx < 4 ) | 5030 if( ctx < 4 ) |
5031 ctx = 4; | 5031 ctx = 4; |
5032 else | 5032 else |
5033 ctx = 5; | 5033 ctx = 5; |
5034 if(ref >= 32 /*h->ref_list[list]*/){ | 5034 if(ref >= 32 /*h->ref_list[list]*/){ |
5035 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n"); | 5035 return -1; |
5036 return 0; //FIXME we should return -1 and check the return everywhere | |
5037 } | 5036 } |
5038 } | 5037 } |
5039 return ref; | 5038 return ref; |
5040 } | 5039 } |
5041 | 5040 |
5545 | 5544 |
5546 for( list = 0; list < h->list_count; list++ ) { | 5545 for( list = 0; list < h->list_count; list++ ) { |
5547 for( i = 0; i < 4; i++ ) { | 5546 for( i = 0; i < 4; i++ ) { |
5548 if(IS_DIRECT(h->sub_mb_type[i])) continue; | 5547 if(IS_DIRECT(h->sub_mb_type[i])) continue; |
5549 if(IS_DIR(h->sub_mb_type[i], 0, list)){ | 5548 if(IS_DIR(h->sub_mb_type[i], 0, list)){ |
5550 if( h->ref_count[list] > 1 ) | 5549 if( h->ref_count[list] > 1 ){ |
5551 ref[list][i] = decode_cabac_mb_ref( h, list, 4*i ); | 5550 ref[list][i] = decode_cabac_mb_ref( h, list, 4*i ); |
5552 else | 5551 if(ref[list][i] >= (unsigned)h->ref_count[list]){ |
5552 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref[list][i], h->ref_count[list]); | |
5553 return -1; | |
5554 } | |
5555 }else | |
5553 ref[list][i] = 0; | 5556 ref[list][i] = 0; |
5554 } else { | 5557 } else { |
5555 ref[list][i] = -1; | 5558 ref[list][i] = -1; |
5556 } | 5559 } |
5557 h->ref_cache[list][ scan8[4*i]+1 ]= | 5560 h->ref_cache[list][ scan8[4*i]+1 ]= |
5630 } else { | 5633 } else { |
5631 int list, mx, my, i, mpx, mpy; | 5634 int list, mx, my, i, mpx, mpy; |
5632 if(IS_16X16(mb_type)){ | 5635 if(IS_16X16(mb_type)){ |
5633 for(list=0; list<h->list_count; list++){ | 5636 for(list=0; list<h->list_count; list++){ |
5634 if(IS_DIR(mb_type, 0, list)){ | 5637 if(IS_DIR(mb_type, 0, list)){ |
5635 const int ref = h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 0 ) : 0; | 5638 int ref; |
5639 if(h->ref_count[list] > 1){ | |
5640 ref= decode_cabac_mb_ref(h, list, 0); | |
5641 if(ref >= (unsigned)h->ref_count[list]){ | |
5642 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]); | |
5643 return -1; | |
5644 } | |
5645 }else | |
5646 ref=0; | |
5636 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, ref, 1); | 5647 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, ref, 1); |
5637 }else | 5648 }else |
5638 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (uint8_t)LIST_NOT_USED, 1); //FIXME factorize and the other fill_rect below too | 5649 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (uint8_t)LIST_NOT_USED, 1); //FIXME factorize and the other fill_rect below too |
5639 } | 5650 } |
5640 for(list=0; list<h->list_count; list++){ | 5651 for(list=0; list<h->list_count; list++){ |
5653 } | 5664 } |
5654 else if(IS_16X8(mb_type)){ | 5665 else if(IS_16X8(mb_type)){ |
5655 for(list=0; list<h->list_count; list++){ | 5666 for(list=0; list<h->list_count; list++){ |
5656 for(i=0; i<2; i++){ | 5667 for(i=0; i<2; i++){ |
5657 if(IS_DIR(mb_type, i, list)){ | 5668 if(IS_DIR(mb_type, i, list)){ |
5658 const int ref= h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 8*i ) : 0; | 5669 int ref; |
5670 if(h->ref_count[list] > 1){ | |
5671 ref= decode_cabac_mb_ref( h, list, 8*i ); | |
5672 if(ref >= (unsigned)h->ref_count[list]){ | |
5673 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]); | |
5674 return -1; | |
5675 } | |
5676 }else | |
5677 ref=0; | |
5659 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, ref, 1); | 5678 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, ref, 1); |
5660 }else | 5679 }else |
5661 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); | 5680 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); |
5662 } | 5681 } |
5663 } | 5682 } |
5680 }else{ | 5699 }else{ |
5681 assert(IS_8X16(mb_type)); | 5700 assert(IS_8X16(mb_type)); |
5682 for(list=0; list<h->list_count; list++){ | 5701 for(list=0; list<h->list_count; list++){ |
5683 for(i=0; i<2; i++){ | 5702 for(i=0; i<2; i++){ |
5684 if(IS_DIR(mb_type, i, list)){ //FIXME optimize | 5703 if(IS_DIR(mb_type, i, list)){ //FIXME optimize |
5685 const int ref= h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 4*i ) : 0; | 5704 int ref; |
5705 if(h->ref_count[list] > 1){ | |
5706 ref= decode_cabac_mb_ref( h, list, 4*i ); | |
5707 if(ref >= (unsigned)h->ref_count[list]){ | |
5708 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]); | |
5709 return -1; | |
5710 } | |
5711 }else | |
5712 ref=0; | |
5686 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, ref, 1); | 5713 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, ref, 1); |
5687 }else | 5714 }else |
5688 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); | 5715 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); |
5689 } | 5716 } |
5690 } | 5717 } |