do_xpsnr() allocated buf_org_m1/buf_org_m2 and, for 8-bit input, the per-component buf_org/buf_rec buffers without checking the results. get_wsse() only validates weights/sse_luma, so a failed allocation of the other buffers led to a NULL pointer dereference (the 8-bit buf_org/buf_rec are written directly before get_wsse() is even called). Return AVERROR(ENOMEM) when any of these allocations fail.
Signed-off-by: Anas Khan <[email protected]> --- libavfilter/vf_xpsnr.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavfilter/vf_xpsnr.c b/libavfilter/vf_xpsnr.c index 34c875456e..38f43fd534 100644 --- a/libavfilter/vf_xpsnr.c +++ b/libavfilter/vf_xpsnr.c @@ -431,6 +431,8 @@ static int do_xpsnr(FFFrameSync *fs) s->buf_org_m1 = av_calloc(s->plane_height[0], stride_org_bpp * sizeof(int16_t)); if (!s->buf_org_m2) s->buf_org_m2 = av_calloc(s->plane_height[0], stride_org_bpp * sizeof(int16_t)); + if (!s->buf_org_m1 || !s->buf_org_m2) + return AVERROR(ENOMEM); if (s->bpp == 1) { /* 8 bit */ for (c = 0; c < s->num_comps; c++) { /* allocate org/rec buffer memory */ @@ -442,6 +444,8 @@ static int do_xpsnr(FFFrameSync *fs) s->buf_org[c] = av_calloc(s->plane_width[c], s->plane_height[c] * sizeof(int16_t)); if (!s->buf_rec[c]) s->buf_rec[c] = av_calloc(s->plane_width[c], s->plane_height[c] * sizeof(int16_t)); + if (!s->buf_org[c] || !s->buf_rec[c]) + return AVERROR(ENOMEM); porg[c] = s->buf_org[c]; prec[c] = s->buf_rec[c]; -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
