Mercurial > libavcodec.hg

comparison h264.c @ 8391:c1eb379daf90 libavcodec

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Check ref values in CABAC H.264 for validity.
author michael
date Thu, 18 Dec 2008 23:31:10 +0000
parents 31f8d52753e4
children ee23da1a0c74
comparison
equal deleted inserted replaced
8390:31f8d52753e4 8391:c1eb379daf90
5030 if( ctx < 4 ) 5030 if( ctx < 4 )
5031 ctx = 4; 5031 ctx = 4;
5032 else 5032 else
5033 ctx = 5; 5033 ctx = 5;
5034 if(ref >= 32 /*h->ref_list[list]*/){ 5034 if(ref >= 32 /*h->ref_list[list]*/){
5035 av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n"); 5035 return -1;
5036 return 0; //FIXME we should return -1 and check the return everywhere
5037 } 5036 }
5038 } 5037 }
5039 return ref; 5038 return ref;
5040 } 5039 }
5041 5040
5545 5544
5546 for( list = 0; list < h->list_count; list++ ) { 5545 for( list = 0; list < h->list_count; list++ ) {
5547 for( i = 0; i < 4; i++ ) { 5546 for( i = 0; i < 4; i++ ) {
5548 if(IS_DIRECT(h->sub_mb_type[i])) continue; 5547 if(IS_DIRECT(h->sub_mb_type[i])) continue;
5549 if(IS_DIR(h->sub_mb_type[i], 0, list)){ 5548 if(IS_DIR(h->sub_mb_type[i], 0, list)){
5550 if( h->ref_count[list] > 1 ) 5549 if( h->ref_count[list] > 1 ){
5551 ref[list][i] = decode_cabac_mb_ref( h, list, 4*i ); 5550 ref[list][i] = decode_cabac_mb_ref( h, list, 4*i );
5552 else 5551 if(ref[list][i] >= (unsigned)h->ref_count[list]){
5552 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref[list][i], h->ref_count[list]);
5553 return -1;
5554 }
5555 }else
5553 ref[list][i] = 0; 5556 ref[list][i] = 0;
5554 } else { 5557 } else {
5555 ref[list][i] = -1; 5558 ref[list][i] = -1;
5556 } 5559 }
5557 h->ref_cache[list][ scan8[4*i]+1 ]= 5560 h->ref_cache[list][ scan8[4*i]+1 ]=
5630 } else { 5633 } else {
5631 int list, mx, my, i, mpx, mpy; 5634 int list, mx, my, i, mpx, mpy;
5632 if(IS_16X16(mb_type)){ 5635 if(IS_16X16(mb_type)){
5633 for(list=0; list<h->list_count; list++){ 5636 for(list=0; list<h->list_count; list++){
5634 if(IS_DIR(mb_type, 0, list)){ 5637 if(IS_DIR(mb_type, 0, list)){
5635 const int ref = h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 0 ) : 0; 5638 int ref;
5639 if(h->ref_count[list] > 1){
5640 ref= decode_cabac_mb_ref(h, list, 0);
5641 if(ref >= (unsigned)h->ref_count[list]){
5642 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]);
5643 return -1;
5644 }
5645 }else
5646 ref=0;
5636 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, ref, 1); 5647 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, ref, 1);
5637 }else 5648 }else
5638 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (uint8_t)LIST_NOT_USED, 1); //FIXME factorize and the other fill_rect below too 5649 fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (uint8_t)LIST_NOT_USED, 1); //FIXME factorize and the other fill_rect below too
5639 } 5650 }
5640 for(list=0; list<h->list_count; list++){ 5651 for(list=0; list<h->list_count; list++){
5653 } 5664 }
5654 else if(IS_16X8(mb_type)){ 5665 else if(IS_16X8(mb_type)){
5655 for(list=0; list<h->list_count; list++){ 5666 for(list=0; list<h->list_count; list++){
5656 for(i=0; i<2; i++){ 5667 for(i=0; i<2; i++){
5657 if(IS_DIR(mb_type, i, list)){ 5668 if(IS_DIR(mb_type, i, list)){
5658 const int ref= h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 8*i ) : 0; 5669 int ref;
5670 if(h->ref_count[list] > 1){
5671 ref= decode_cabac_mb_ref( h, list, 8*i );
5672 if(ref >= (unsigned)h->ref_count[list]){
5673 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]);
5674 return -1;
5675 }
5676 }else
5677 ref=0;
5659 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, ref, 1); 5678 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, ref, 1);
5660 }else 5679 }else
5661 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1); 5680 fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1);
5662 } 5681 }
5663 } 5682 }
5680 }else{ 5699 }else{
5681 assert(IS_8X16(mb_type)); 5700 assert(IS_8X16(mb_type));
5682 for(list=0; list<h->list_count; list++){ 5701 for(list=0; list<h->list_count; list++){
5683 for(i=0; i<2; i++){ 5702 for(i=0; i<2; i++){
5684 if(IS_DIR(mb_type, i, list)){ //FIXME optimize 5703 if(IS_DIR(mb_type, i, list)){ //FIXME optimize
5685 const int ref= h->ref_count[list] > 1 ? decode_cabac_mb_ref( h, list, 4*i ) : 0; 5704 int ref;
5705 if(h->ref_count[list] > 1){
5706 ref= decode_cabac_mb_ref( h, list, 4*i );
5707 if(ref >= (unsigned)h->ref_count[list]){
5708 av_log(s->avctx, AV_LOG_ERROR, "Reference %d >= %d\n", ref, h->ref_count[list]);
5709 return -1;
5710 }
5711 }else
5712 ref=0;
5686 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, ref, 1); 5713 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, ref, 1);
5687 }else 5714 }else
5688 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1); 5715 fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1);
5689 } 5716 }
5690 } 5717 }