comparison mpegvideo.c @ 809:79de6308c34d libavcodec

fixing another assert q>0.0 issue caused by variance < 0, this fix allso changes the inter/intra decission very slightly -> all regression checksums need to be updated

808:e9bfaabcf07d

                    int yy = mb_y * 16;
                    uint8_t *pix = s->new_picture[0] + (yy * s->linesize) + xx;
                    int varc;
                    int sum = pix_sum(pix, s->linesize);
    
                    sum= (sum+8)>>4;
                    varc = (pix_norm1(pix, s->linesize) - sum*sum + 500 + 128)>>8;

                    s->mb_var [s->mb_width * mb_y + mb_x] = varc;
                    s->mb_mean[s->mb_width * mb_y + mb_x] = (sum+7)>>4;
                    s->mb_var_sum    += varc;
                }
            }
        }
    }

809:79de6308c34d

                    int yy = mb_y * 16;
                    uint8_t *pix = s->new_picture[0] + (yy * s->linesize) + xx;
                    int varc;
                    int sum = pix_sum(pix, s->linesize);
    
                    varc = (pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;

                    s->mb_var [s->mb_width * mb_y + mb_x] = varc;
                    s->mb_mean[s->mb_width * mb_y + mb_x] = (sum+128)>>8;
                    s->mb_var_sum    += varc;
                }
            }
        }
    }