Mercurial > libavcodec.hg

comparison utils.c @ 2950:2c4753d27834 libavcodec

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
default_get_buffer() cleanup fixes probably exploitable heap overflow heap overflow found by (Simon Kilvington <s D kilvington A eris D qinetiq D com>)
author michael
date Fri, 02 Dec 2005 00:12:37 +0000
parents d268a7c6c91d
children ef2149182f1c
comparison
equal deleted inserted replaced
2949:e5a10ae14ffb 2950:2c4753d27834
290 if(buf->base[0]){ 290 if(buf->base[0]){
291 pic->age= *picture_number - buf->last_pic_num; 291 pic->age= *picture_number - buf->last_pic_num;
292 buf->last_pic_num= *picture_number; 292 buf->last_pic_num= *picture_number;
293 }else{ 293 }else{
294 int h_chroma_shift, v_chroma_shift; 294 int h_chroma_shift, v_chroma_shift;
295 int pixel_size; 295 int pixel_size, size[3];
296 296 AVPicture picture;
297
297 avcodec_get_chroma_sub_sample(s->pix_fmt, &h_chroma_shift, &v_chroma_shift); 298 avcodec_get_chroma_sub_sample(s->pix_fmt, &h_chroma_shift, &v_chroma_shift);
298
299 switch(s->pix_fmt){
300 case PIX_FMT_RGB555:
301 case PIX_FMT_RGB565:
302 case PIX_FMT_YUV422:
303 case PIX_FMT_UYVY422:
304 pixel_size=2;
305 break;
306 case PIX_FMT_RGB24:
307 case PIX_FMT_BGR24:
308 pixel_size=3;
309 break;
310 case PIX_FMT_RGBA32:
311 pixel_size=4;
312 break;
313 default:
314 pixel_size=1;
315 }
316 299
317 avcodec_align_dimensions(s, &w, &h); 300 avcodec_align_dimensions(s, &w, &h);
318 301
319 if(!(s->flags&CODEC_FLAG_EMU_EDGE)){ 302 if(!(s->flags&CODEC_FLAG_EMU_EDGE)){
320 w+= EDGE_WIDTH*2; 303 w+= EDGE_WIDTH*2;
321 h+= EDGE_WIDTH*2; 304 h+= EDGE_WIDTH*2;
322 } 305 }
323 306 avpicture_fill(&picture, NULL, s->pix_fmt, w, h);
307 pixel_size= picture.linesize[0]*8 / w;
308 //av_log(NULL, AV_LOG_ERROR, "%d %d %d %d\n", (int)picture.data[1], w, h, s->pix_fmt);
309 assert(pixel_size>=1);
310 //FIXME next ensures that linesize= 2^x uvlinesize, thats needed because some MC code assumes it
311 if(pixel_size == 3*8)
312 w= ALIGN(w, STRIDE_ALIGN<<h_chroma_shift);
313 else
314 w= ALIGN(pixel_size*w, STRIDE_ALIGN<<(h_chroma_shift+3)) / pixel_size;
315 size[1] = avpicture_fill(&picture, NULL, s->pix_fmt, w, h);
316 size[0] = picture.linesize[0] * h;
317 size[1] -= size[0];
318 if(picture.data[2])
319 size[1]= size[2]= size[1]/2;
320 else
321 size[2]= 0;
322
324 buf->last_pic_num= -256*256*256*64; 323 buf->last_pic_num= -256*256*256*64;
325 324 memset(buf->base, 0, sizeof(buf->base));
326 for(i=0; i<3; i++){ 325 memset(buf->data, 0, sizeof(buf->data));
326
327 for(i=0; i<3 && size[i]; i++){
327 const int h_shift= i==0 ? 0 : h_chroma_shift; 328 const int h_shift= i==0 ? 0 : h_chroma_shift;
328 const int v_shift= i==0 ? 0 : v_chroma_shift; 329 const int v_shift= i==0 ? 0 : v_chroma_shift;
329 330
330 //FIXME next ensures that linesize= 2^x uvlinesize, thats needed because some MC code assumes it 331 buf->linesize[i]= picture.linesize[i];
331 buf->linesize[i]= ALIGN(pixel_size*w>>h_shift, STRIDE_ALIGN<<(h_chroma_shift-h_shift)); 332
332 333 buf->base[i]= av_malloc(size[i]+16); //FIXME 16
333 buf->base[i]= av_malloc((buf->linesize[i]*h>>v_shift)+16); //FIXME 16
334 if(buf->base[i]==NULL) return -1; 334 if(buf->base[i]==NULL) return -1;
335 memset(buf->base[i], 128, buf->linesize[i]*h>>v_shift); 335 memset(buf->base[i], 128, size[i]);
336 336
337 if(s->flags&CODEC_FLAG_EMU_EDGE) 337 // no edge if EDEG EMU or not planar YUV, we check for PAL8 redundantly to protect against a exploitable bug regression ...
338 if((s->flags&CODEC_FLAG_EMU_EDGE) || (s->pix_fmt == PIX_FMT_PAL8) || !size[2])
338 buf->data[i] = buf->base[i]; 339 buf->data[i] = buf->base[i];
339 else 340 else
340 buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), STRIDE_ALIGN); 341 buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), STRIDE_ALIGN);
341 } 342 }
342 pic->age= 256*256*256*64; 343 pic->age= 256*256*256*64;