From 107ec830427f0a8f8e75787f4afa23f084e03029 Mon Sep 17 00:00:00 2001 From: Ronald S. Bultje Date: Sun, 4 Dec 2011 14:08:03 -0800 Subject: [PATCH] Fix overflow in swscale. --- libswscale/swscale.c | 37 ++++++++++++++++++++++++++++++++++--- libswscale/utils.c | 12 +++++++----- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 975a0bd..3d41b8d 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -2486,9 +2486,9 @@ static int swScale(SwsContext *c, const uint8_t* src[], const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<chrDstVSubSample) - 1), dstH-1)]; const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input - int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input - int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input - int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input + int lastLumSrcY = FFMIN(c->srcH, firstLumSrcY + vLumFilterSize) - 1; // Last line needed as input + int lastLumSrcY2 = FFMIN(c->srcH, firstLumSrcY2 + vLumFilterSize) - 1; // Last line needed as input + int lastChrSrcY = FFMIN(c->chrSrcH, firstChrSrcY + vChrFilterSize) - 1; // Last line needed as input int enough_lines; //handle holes (FAST_BILINEAR & weird filters) @@ -2580,10 +2580,41 @@ static int swScale(SwsContext *c, const uint8_t* src[], } { + const int16_t *tmpYA[2][vLumBufSize], *tmpUV[2][vChrBufSize]; const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize; const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize; const int16_t **chrVSrcPtr= (const int16_t **) chrVPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize; const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL; + + if (firstLumSrcY < 0 || firstLumSrcY + vLumFilterSize > c->srcH) { + //printf("Overflow %d %d for [0, %d]\n", firstLumSrcY, + // firstLumSrcY + vLumFilterSize - 1, c->srcH); + int neg = -firstLumSrcY, i, end = FFMIN(c->srcH - firstLumSrcY, vLumFilterSize); + for (i = 0; i < neg; i++) tmpYA[0][i] = lumSrcPtr[neg]; + for ( ; i < end; i++) tmpYA[0][i] = lumSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) tmpYA[0][i] = tmpYA[0][i-1]; + lumSrcPtr = tmpYA[0]; + if (alpSrcPtr) { + for (i = 0; i < neg; i++) tmpYA[1][i] = alpSrcPtr[neg]; + for ( ; i < end; i++) tmpYA[1][i] = alpSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) tmpYA[1][i] = tmpYA[1][i - 1]; + alpSrcPtr = tmpYA[1]; + } + } + if (firstChrSrcY < 0 || firstChrSrcY + vChrFilterSize > c->chrSrcH) { + //printf("Overflow %d %d for [0, %d]\n", firstChrSrcY, + // firstChrSrcY + vChrFilterSize - 1, c->chrSrcH); + int neg = -firstChrSrcY, i, end = FFMIN(c->chrSrcH - firstChrSrcY, vChrFilterSize); + for (i = 0; i < neg; i++) { tmpUV[0][i] = chrUSrcPtr[neg]; + tmpUV[1][i] = chrVSrcPtr[neg]; } + for ( ; i < end; i++) { tmpUV[0][i] = chrUSrcPtr[i]; + tmpUV[1][i] = chrVSrcPtr[i]; } + for ( ; i < vChrFilterSize; i++) { tmpUV[0][i] = tmpUV[0][i - 1]; + tmpUV[1][i] = tmpUV[1][i - 1]; } + chrUSrcPtr = tmpUV[0]; + chrVSrcPtr = tmpUV[1]; + } + if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like const int chrSkipMask= (1<chrDstVSubSample)-1; diff --git a/libswscale/utils.c b/libswscale/utils.c index b644ed9..55500de 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -182,7 +182,7 @@ static double getSplineCoeff(double a, double b, double c, double d, double dist static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSize, int xInc, int srcW, int dstW, int filterAlign, int one, int flags, int cpu_flags, - SwsVector *srcFilter, SwsVector *dstFilter, double param[2]) + SwsVector *srcFilter, SwsVector *dstFilter, double param[2], int is_horizontal) { int i; int filterSize; @@ -459,6 +459,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi //FIXME try to align filterPos if possible //fix borders + if (is_horizontal) { for (i=0; ihLumFilter, &c->hLumFilterPos, &c->hLumFilterSize, c->lumXInc, srcW , dstW, filterAlign, 1<<14, (flags&SWS_BICUBLIN) ? (flags|SWS_BICUBIC) : flags, cpu_flags, - srcFilter->lumH, dstFilter->lumH, c->param) < 0) + srcFilter->lumH, dstFilter->lumH, c->param, 1) < 0) goto fail; if (initFilter(&c->hChrFilter, &c->hChrFilterPos, &c->hChrFilterSize, c->chrXInc, c->chrSrcW, c->chrDstW, filterAlign, 1<<14, (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags, cpu_flags, - srcFilter->chrH, dstFilter->chrH, c->param) < 0) + srcFilter->chrH, dstFilter->chrH, c->param, 1) < 0) goto fail; } } // initialize horizontal stuff @@ -978,12 +980,12 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) if (initFilter(&c->vLumFilter, &c->vLumFilterPos, &c->vLumFilterSize, c->lumYInc, srcH , dstH, filterAlign, (1<<12), (flags&SWS_BICUBLIN) ? (flags|SWS_BICUBIC) : flags, cpu_flags, - srcFilter->lumV, dstFilter->lumV, c->param) < 0) + srcFilter->lumV, dstFilter->lumV, c->param, 0) < 0) goto fail; if (initFilter(&c->vChrFilter, &c->vChrFilterPos, &c->vChrFilterSize, c->chrYInc, c->chrSrcH, c->chrDstH, filterAlign, (1<<12), (flags&SWS_BICUBLIN) ? (flags|SWS_BILINEAR) : flags, cpu_flags, - srcFilter->chrV, dstFilter->chrV, c->param) < 0) + srcFilter->chrV, dstFilter->chrV, c->param, 0) < 0) goto fail; #if HAVE_ALTIVEC -- 1.7.2.1