comparison libswscale/swscale.c @ 29542:3e7ffd70b12b

swscale: Check for return values of malloc.

29541:b5a8ade9175b

#include "swscale.h"
#include "swscale_internal.h"
#include "rgb2rgb.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/x86_cpu.h"
#include "libavutil/bswap.h"

unsigned swscale_version(void)
{
    return LIBSWSCALE_VERSION_INT;

    if (flags & SWS_CPU_CAPS_MMX)
        __asm__ volatile("emms\n\t"::: "memory"); //FIXME this should not be required but it IS (even for non-MMX versions)
#endif

    // NOTE: the +1 is for the MMX scaler which reads over the end
    *filterPos = av_malloc((dstW+1)*sizeof(int16_t));

    if (FFABS(xInc - 0x10000) <10) { // unscaled
        int i;
        filterSize= 1;
        filter= av_mallocz(dstW*sizeof(*filter)*filterSize);

        for (i=0; i<dstW; i++) {
            filter[i*filterSize]= fone;
            (*filterPos)[i]=i;
        }

    } else if (flags&SWS_POINT) { // lame looking point sampling mode
        int i;
        int xDstInSrc;
        filterSize= 1;
        filter= av_malloc(dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc/2 - 0x8000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-1)<<15) + (1<<15))>>16;

        }
    } else if ((xInc <= (1<<16) && (flags&SWS_AREA)) || (flags&SWS_FAST_BILINEAR)) { // bilinear upscale
        int i;
        int xDstInSrc;
        filterSize= 2;
        filter= av_malloc(dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc/2 - 0x8000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-1)<<15) + (1<<15))>>16;
            int j;

        if (xInc <= 1<<16)      filterSize= 1 + sizeFactor; // upscale
        else                    filterSize= 1 + (sizeFactor*srcW + dstW - 1)/ dstW;

        if (filterSize > srcW-2) filterSize=srcW-2;

        filter= av_malloc(dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc - 0x10000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-2)<<16)) / (1<<17);
            int j;

    assert(filterSize>0);
    filter2Size= filterSize;
    if (srcFilter) filter2Size+= srcFilter->length - 1;
    if (dstFilter) filter2Size+= dstFilter->length - 1;
    assert(filter2Size>0);
    filter2= av_mallocz(filter2Size*dstW*sizeof(*filter2));

    for (i=0; i<dstW; i++) {
        int j, k;

        if(srcFilter) {

    assert(minFilterSize > 0);
    filterSize= (minFilterSize +(filterAlign-1)) & (~(filterAlign-1));
    assert(filterSize > 0);
    filter= av_malloc(filterSize*dstW*sizeof(*filter));
    if (filterSize >= MAX_FILTER_SIZE*16/((flags&SWS_ACCURATE_RND) ? APCK_SIZE : 16) || !filter)
        goto error;
    *outFilterSize= filterSize;

    if (flags&SWS_PRINT_INFO)
        av_log(NULL, AV_LOG_VERBOSE, "SwScaler: reducing / aligning filtersize %d -> %d\n", filter2Size, filterSize);
    /* try to reduce the filter-size (step2 reduce it) */

        }
    }

    // Note the +1 is for the MMX scaler which reads over the end
    /* align at 16 for AltiVec (needed by hScale_altivec_real) */
    *outFilter= av_mallocz(*outFilterSize*(dstW+1)*sizeof(int16_t));

    /* normalize & store in outFilter */
    for (i=0; i<dstW; i++) {
        int j;
        int64_t error=0;

        int j= dstW*(*outFilterSize);
        (*outFilter)[j + i]= (*outFilter)[j + i - (*outFilterSize)];
    }

    ret=0;
error:
    av_free(filter);
    av_free(filter2);
    return ret;
}

    }

    if (!dstFilter) dstFilter= &dummyFilter;
    if (!srcFilter) srcFilter= &dummyFilter;

    c= av_mallocz(sizeof(SwsContext));

    c->av_class = &sws_context_class;
    c->srcW= srcW;
    c->srcH= srcH;
    c->dstW= dstW;

#else
            c->lumMmx2FilterCode = av_malloc(c->lumMmx2FilterCodeSize);
            c->chrMmx2FilterCode = av_malloc(c->chrMmx2FilterCodeSize);
#endif

            c->lumMmx2Filter   = av_malloc((dstW        /8+8)*sizeof(int16_t));
            c->chrMmx2Filter   = av_malloc((c->chrDstW  /4+8)*sizeof(int16_t));
            c->lumMmx2FilterPos= av_malloc((dstW      /2/8+8)*sizeof(int32_t));
            c->chrMmx2FilterPos= av_malloc((c->chrDstW/2/4+8)*sizeof(int32_t));

            initMMX2HScaler(      dstW, c->lumXInc, c->lumMmx2FilterCode, c->lumMmx2Filter, c->lumMmx2FilterPos, 8);
            initMMX2HScaler(c->chrDstW, c->chrXInc, c->chrMmx2FilterCode, c->chrMmx2Filter, c->chrMmx2FilterPos, 4);

#ifdef MAP_ANONYMOUS

                   c->chrSrcH, c->chrDstH, filterAlign, (1<<12),
                   (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags,
                   srcFilter->chrV, dstFilter->chrV, c->param);

#ifdef COMPILE_ALTIVEC
        c->vYCoeffsBank = av_malloc(sizeof (vector signed short)*c->vLumFilterSize*c->dstH);
        c->vCCoeffsBank = av_malloc(sizeof (vector signed short)*c->vChrFilterSize*c->chrDstH);

        for (i=0;i<c->vLumFilterSize*c->dstH;i++) {
            int j;
            short *p = (short *)&c->vYCoeffsBank[i];
            for (j=0;j<8;j++)

            c->vChrBufSize= (nextSlice>>c->chrSrcVSubSample) - c->vChrFilterPos[chrI];
    }

    // allocate pixbufs (we use dynamic allocation because otherwise we would need to
    // allocate several megabytes to handle all possible cases)
    c->lumPixBuf= av_malloc(c->vLumBufSize*2*sizeof(int16_t*));
    c->chrPixBuf= av_malloc(c->vChrBufSize*2*sizeof(int16_t*));
    if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat) && isALPHA(c->dstFormat))
        c->alpPixBuf= av_malloc(c->vLumBufSize*2*sizeof(int16_t*));
    //Note we need at least one pixel more at the end because of the MMX code (just in case someone wanna replace the 4000/8000)
    /* align at 16 bytes for AltiVec */
    for (i=0; i<c->vLumBufSize; i++)
        c->lumPixBuf[i]= c->lumPixBuf[i+c->vLumBufSize]= av_mallocz(VOF+1);
    for (i=0; i<c->vChrBufSize; i++)
        c->chrPixBuf[i]= c->chrPixBuf[i+c->vChrBufSize]= av_malloc((VOF+1)*2);
    if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf)
        for (i=0; i<c->vLumBufSize; i++)
            c->alpPixBuf[i]= c->alpPixBuf[i+c->vLumBufSize]= av_mallocz(VOF+1);

    //try to avoid drawing green stuff between the right end and the stride end
    for (i=0; i<c->vChrBufSize; i++) memset(c->chrPixBuf[i], 64, (VOF+1)*2);

    assert(2*VOFW == VOF);

               c->chrSrcW, c->chrSrcH, c->chrDstW, c->chrDstH, c->chrXInc, c->chrYInc);
    }

    c->swScale= getSwsFunc(c);
    return c;
}

static void reset_ptr(uint8_t* src[], int format)
{
    if(!isALPHA(format))

                                float lumaSharpen, float chromaSharpen,
                                float chromaHShift, float chromaVShift,
                                int verbose)
{
    SwsFilter *filter= av_malloc(sizeof(SwsFilter));

    if (lumaGBlur!=0.0) {
        filter->lumH= sws_getGaussianVec(lumaGBlur, 3.0);
        filter->lumV= sws_getGaussianVec(lumaGBlur, 3.0);
    } else {

29542:3e7ffd70b12b

#include "swscale.h"
#include "swscale_internal.h"
#include "rgb2rgb.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/x86_cpu.h"
#include "libavutil/avutil.h"
#include "libavutil/bswap.h"

unsigned swscale_version(void)
{
    return LIBSWSCALE_VERSION_INT;

    if (flags & SWS_CPU_CAPS_MMX)
        __asm__ volatile("emms\n\t"::: "memory"); //FIXME this should not be required but it IS (even for non-MMX versions)
#endif

    // NOTE: the +1 is for the MMX scaler which reads over the end
    CHECKED_ALLOC(*filterPos, (dstW+1)*sizeof(int16_t));

    if (FFABS(xInc - 0x10000) <10) { // unscaled
        int i;
        filterSize= 1;
        CHECKED_ALLOCZ(filter, dstW*sizeof(*filter)*filterSize);

        for (i=0; i<dstW; i++) {
            filter[i*filterSize]= fone;
            (*filterPos)[i]=i;
        }

    } else if (flags&SWS_POINT) { // lame looking point sampling mode
        int i;
        int xDstInSrc;
        filterSize= 1;
        CHECKED_ALLOC(filter, dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc/2 - 0x8000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-1)<<15) + (1<<15))>>16;

        }
    } else if ((xInc <= (1<<16) && (flags&SWS_AREA)) || (flags&SWS_FAST_BILINEAR)) { // bilinear upscale
        int i;
        int xDstInSrc;
        filterSize= 2;
        CHECKED_ALLOC(filter, dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc/2 - 0x8000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-1)<<15) + (1<<15))>>16;
            int j;

        if (xInc <= 1<<16)      filterSize= 1 + sizeFactor; // upscale
        else                    filterSize= 1 + (sizeFactor*srcW + dstW - 1)/ dstW;

        if (filterSize > srcW-2) filterSize=srcW-2;

        CHECKED_ALLOC(filter, dstW*sizeof(*filter)*filterSize);

        xDstInSrc= xInc - 0x10000;
        for (i=0; i<dstW; i++) {
            int xx= (xDstInSrc - ((filterSize-2)<<16)) / (1<<17);
            int j;

    assert(filterSize>0);
    filter2Size= filterSize;
    if (srcFilter) filter2Size+= srcFilter->length - 1;
    if (dstFilter) filter2Size+= dstFilter->length - 1;
    assert(filter2Size>0);
    CHECKED_ALLOCZ(filter2, filter2Size*dstW*sizeof(*filter2));

    for (i=0; i<dstW; i++) {
        int j, k;

        if(srcFilter) {

    assert(minFilterSize > 0);
    filterSize= (minFilterSize +(filterAlign-1)) & (~(filterAlign-1));
    assert(filterSize > 0);
    filter= av_malloc(filterSize*dstW*sizeof(*filter));
    if (filterSize >= MAX_FILTER_SIZE*16/((flags&SWS_ACCURATE_RND) ? APCK_SIZE : 16) || !filter)
        goto fail;
    *outFilterSize= filterSize;

    if (flags&SWS_PRINT_INFO)
        av_log(NULL, AV_LOG_VERBOSE, "SwScaler: reducing / aligning filtersize %d -> %d\n", filter2Size, filterSize);
    /* try to reduce the filter-size (step2 reduce it) */

        }
    }

    // Note the +1 is for the MMX scaler which reads over the end
    /* align at 16 for AltiVec (needed by hScale_altivec_real) */
    CHECKED_ALLOCZ(*outFilter, *outFilterSize*(dstW+1)*sizeof(int16_t));

    /* normalize & store in outFilter */
    for (i=0; i<dstW; i++) {
        int j;
        int64_t error=0;

        int j= dstW*(*outFilterSize);
        (*outFilter)[j + i]= (*outFilter)[j + i - (*outFilterSize)];
    }

    ret=0;
fail:
    av_free(filter);
    av_free(filter2);
    return ret;
}

    }

    if (!dstFilter) dstFilter= &dummyFilter;
    if (!srcFilter) srcFilter= &dummyFilter;

    CHECKED_ALLOCZ(c, sizeof(SwsContext));

    c->av_class = &sws_context_class;
    c->srcW= srcW;
    c->srcH= srcH;
    c->dstW= dstW;

#else
            c->lumMmx2FilterCode = av_malloc(c->lumMmx2FilterCodeSize);
            c->chrMmx2FilterCode = av_malloc(c->chrMmx2FilterCodeSize);
#endif

            CHECKED_ALLOCZ(c->lumMmx2Filter   , (dstW        /8+8)*sizeof(int16_t));
            CHECKED_ALLOCZ(c->chrMmx2Filter   , (c->chrDstW  /4+8)*sizeof(int16_t));
            CHECKED_ALLOCZ(c->lumMmx2FilterPos, (dstW      /2/8+8)*sizeof(int32_t));
            CHECKED_ALLOCZ(c->chrMmx2FilterPos, (c->chrDstW/2/4+8)*sizeof(int32_t));

            initMMX2HScaler(      dstW, c->lumXInc, c->lumMmx2FilterCode, c->lumMmx2Filter, c->lumMmx2FilterPos, 8);
            initMMX2HScaler(c->chrDstW, c->chrXInc, c->chrMmx2FilterCode, c->chrMmx2Filter, c->chrMmx2FilterPos, 4);

#ifdef MAP_ANONYMOUS

                   c->chrSrcH, c->chrDstH, filterAlign, (1<<12),
                   (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags,
                   srcFilter->chrV, dstFilter->chrV, c->param);

#ifdef COMPILE_ALTIVEC
        CHECKED_ALLOC(c->vYCoeffsBank, sizeof (vector signed short)*c->vLumFilterSize*c->dstH);
        CHECKED_ALLOC(c->vCCoeffsBank, sizeof (vector signed short)*c->vChrFilterSize*c->chrDstH);

        for (i=0;i<c->vLumFilterSize*c->dstH;i++) {
            int j;
            short *p = (short *)&c->vYCoeffsBank[i];
            for (j=0;j<8;j++)

            c->vChrBufSize= (nextSlice>>c->chrSrcVSubSample) - c->vChrFilterPos[chrI];
    }

    // allocate pixbufs (we use dynamic allocation because otherwise we would need to
    // allocate several megabytes to handle all possible cases)
    CHECKED_ALLOC(c->lumPixBuf, c->vLumBufSize*2*sizeof(int16_t*));
    CHECKED_ALLOC(c->chrPixBuf, c->vChrBufSize*2*sizeof(int16_t*));
    if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat) && isALPHA(c->dstFormat))
        CHECKED_ALLOCZ(c->alpPixBuf, c->vLumBufSize*2*sizeof(int16_t*));
    //Note we need at least one pixel more at the end because of the MMX code (just in case someone wanna replace the 4000/8000)
    /* align at 16 bytes for AltiVec */
    for (i=0; i<c->vLumBufSize; i++)
    {
        CHECKED_ALLOCZ(c->lumPixBuf[i+c->vLumBufSize], VOF+1);
        c->lumPixBuf[i] = c->lumPixBuf[i+c->vLumBufSize];
    }
    for (i=0; i<c->vChrBufSize; i++)
    {
        CHECKED_ALLOC(c->chrPixBuf[i+c->vChrBufSize], (VOF+1)*2);
        c->chrPixBuf[i] = c->chrPixBuf[i+c->vChrBufSize];
    }
    if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf)
        for (i=0; i<c->vLumBufSize; i++)
        {
            CHECKED_ALLOCZ(c->alpPixBuf[i+c->vLumBufSize], VOF+1);
            c->alpPixBuf[i] = c->alpPixBuf[i+c->vLumBufSize];
        }

    //try to avoid drawing green stuff between the right end and the stride end
    for (i=0; i<c->vChrBufSize; i++) memset(c->chrPixBuf[i], 64, (VOF+1)*2);

    assert(2*VOFW == VOF);

               c->chrSrcW, c->chrSrcH, c->chrDstW, c->chrDstH, c->chrXInc, c->chrYInc);
    }

    c->swScale= getSwsFunc(c);
    return c;

fail:
    sws_freeContext(c);
    return NULL;
}

static void reset_ptr(uint8_t* src[], int format)
{
    if(!isALPHA(format))

                                float lumaSharpen, float chromaSharpen,
                                float chromaHShift, float chromaVShift,
                                int verbose)
{
    SwsFilter *filter= av_malloc(sizeof(SwsFilter));
    if (!filter)
        return NULL;

    if (lumaGBlur!=0.0) {
        filter->lumH= sws_getGaussianVec(lumaGBlur, 3.0);
        filter->lumV= sws_getGaussianVec(lumaGBlur, 3.0);
    } else {