PR #23855 opened by ffmpeg-devel URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23855 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23855.patch
**Backport:** https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23742 D3D12VA fully trusts the return value of ff_d3d12va_get_suitable_max_bitstream_size(), which just returns av_image_get_buffer_size(frames_ctx->sw_format, avctx->coded_width, avctx->coded_height, 1). No size checking was done. So a file specifically crafted to bust that limit is able to overflow that buffer. At least I wasn't able to find any size checking done anywhere, so this is what this series adds. This was originally found by depthfirst, but only the instance the first commit fixes. When I looked into it more, I found the same issue in all other d3d12va decoders, as well as the other instance inside of the AV1 one. >From 60aca6ace74384709bd921da7a735ec1cb6b36a2 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Wed, 8 Jul 2026 15:19:33 +0200 Subject: [PATCH 1/7] avcodec/d3d12va_av1: check size of frame bitstream against available buffer size (cherry picked from commit e1ef95c779a3757d21ab970c06aac3e9e1b71974) --- libavcodec/d3d12va_av1.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index fdc7195d12..a3ddf0495a 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -112,6 +112,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const AV1DecContext *h = avctx->priv_data; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; void *mapped_data; @@ -121,6 +122,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU args->Size = sizeof(DXVA_Tile_AV1) * ctx_pic->tile_count; args->pData = ctx_pic->tiles; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + input_args->CompressedBitstream = (D3D12_VIDEO_DECODE_COMPRESSED_BITSTREAM){ .pBuffer = buffer, .Offset = 0, -- 2.52.0 >From a90fbb6bc9fcf1df87bfbfc8a7818fd945d5397e Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:29:01 +0200 Subject: [PATCH 2/7] avcodec/d3d12va_av1: check slice bitstream size against available buffer size (cherry picked from commit 7309ad9fd78d875e60427daa52260834e27ef2ea) --- libavcodec/d3d12va_av1.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index a3ddf0495a..bcf28ed590 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -33,6 +33,7 @@ typedef struct D3D12AV1DecodeContext { D3D12VADecodeContext ctx; uint8_t *bitstream_buffer; + size_t bitstream_size; } D3D12AV1DecodeContext; #define D3D12_AV1_DECODE_CONTEXT(avctx) ((D3D12AV1DecodeContext *)D3D12VA_DECODE_CONTEXT(avctx)) @@ -77,6 +78,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, const AV1DecContext *h = avctx->priv_data; const AV1RawFrameHeader *frame_header = h->raw_frame_header; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; + D3D12AV1DecodeContext *av1_ctx = D3D12_AV1_DECODE_CONTEXT(avctx); int offset = 0; uint32_t tg_start, tg_end; @@ -91,7 +93,11 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, ctx_pic->bitstream = (uint8_t *)buffer; ctx_pic->bitstream_size = size; } else { - ctx_pic->bitstream = D3D12_AV1_DECODE_CONTEXT(avctx)->bitstream_buffer; + if (ctx_pic->bitstream_size + (uint64_t)size > av1_ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Slice bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + ctx_pic->bitstream = av1_ctx->bitstream_buffer; memcpy(ctx_pic->bitstream + ctx_pic->bitstream_size, buffer, size); tg_start = h->tg_start; tg_end = h->tg_end; @@ -180,7 +186,8 @@ static av_cold int d3d12va_av1_decode_init(AVCodecContext *avctx) return ret; if (!av1_ctx->bitstream_buffer) { - av1_ctx->bitstream_buffer = av_malloc(ff_d3d12va_get_suitable_max_bitstream_size(avctx)); + av1_ctx->bitstream_size = ff_d3d12va_get_suitable_max_bitstream_size(avctx); + av1_ctx->bitstream_buffer = av_malloc(av1_ctx->bitstream_size); if (!av1_ctx->bitstream_buffer) return AVERROR(ENOMEM); } @@ -194,6 +201,7 @@ static av_cold int d3d12va_av1_decode_uninit(AVCodecContext *avctx) if (ctx->bitstream_buffer) av_freep(&ctx->bitstream_buffer); + ctx->bitstream_size = 0; return ff_d3d12va_decode_uninit(avctx); } -- 2.52.0 >From 3a44bddfb9ad1416cd7bb02e4977009cfc2dfde9 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:38:37 +0200 Subject: [PATCH 3/7] avcodec/d3d12va_h264: check size of frame bitstream against available buffer size (cherry picked from commit 19e81f9260e64c43604556cbd4079fadee4ac4f1) --- libavcodec/d3d12va_h264.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_h264.c b/libavcodec/d3d12va_h264.c index dec9344aad..5087e480f8 100644 --- a/libavcodec/d3d12va_h264.c +++ b/libavcodec/d3d12va_h264.c @@ -105,6 +105,7 @@ static int d3d12va_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const H264Context *h = avctx->priv_data; const H264Picture *current_picture = h->cur_pic_ptr; H264DecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -113,6 +114,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_H264_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -127,14 +129,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + (uint64_t)size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From b19b023f2a4e7269c7761d653f96ba46bccb24ab Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:40:32 +0200 Subject: [PATCH 4/7] avcodec/d3d12va_hevc: check size of frame bitstream against available buffer size (cherry picked from commit fea3f13a1cc5395b0ab6db95dd20e7cf6023c311) --- libavcodec/d3d12va_hevc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_hevc.c b/libavcodec/d3d12va_hevc.c index e72d49b7d9..1bf1faf9cb 100644 --- a/libavcodec/d3d12va_hevc.c +++ b/libavcodec/d3d12va_hevc.c @@ -101,6 +101,7 @@ static int d3d12va_hevc_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const HEVCContext *h = avctx->priv_data; const HEVCFrame *current_picture = h->cur_frame; HEVCDecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -109,6 +110,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_HEVC_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -123,14 +125,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + (uint64_t)size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 865374b1abc444df702aee831ca404ebf9d71d13 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:42:38 +0200 Subject: [PATCH 5/7] avcodec/d3d12va_mpeg2: check size of frame bitstream against available buffer size (cherry picked from commit 26a9f9b3aeb054901b69518223af0d6f4f991fd9) --- libavcodec/d3d12va_mpeg2.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_mpeg2.c b/libavcodec/d3d12va_mpeg2.c index 47e453dd5e..de9f0c71d1 100644 --- a/libavcodec/d3d12va_mpeg2.c +++ b/libavcodec/d3d12va_mpeg2.c @@ -90,6 +90,7 @@ static int d3d12va_mpeg2_decode_slice(AVCodecContext *avctx, const uint8_t *buff static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const MpegEncContext *s = avctx->priv_data; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -105,6 +106,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU .End = ctx_pic->bitstream_size, }; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, &range, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 >From ae0feeac8d2f87370eaf09bd8e01ecffd01373de Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:20 +0200 Subject: [PATCH 6/7] avcodec/d3d12va_vc1: check size of frame bitstream against available buffer size (cherry picked from commit 1307db3d3c95223b1d2164844e0542cddbb2a401) --- libavcodec/d3d12va_vc1.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_vc1.c b/libavcodec/d3d12va_vc1.c index e64a8e0630..2386d2a29b 100644 --- a/libavcodec/d3d12va_vc1.c +++ b/libavcodec/d3d12va_vc1.c @@ -94,6 +94,7 @@ static int d3d12va_vc1_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VC1Context *v = avctx->priv_data; const MpegEncContext *s = &v->s; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -101,6 +102,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU const unsigned mb_count = s->mb_width * (s->mb_height >> v->field_mode); uint8_t *mapped_data, *mapped_ptr; + UINT bitstream_size = ctx->bitstream_size; static const uint8_t start_code[] = { 0, 0, 1, 0x0d }; @@ -115,6 +117,12 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU unsigned position = slice->dwSliceDataLocation; unsigned size = slice->dwSliceBitsInBuffer / 8; + if ((uint64_t)size + ((avctx->codec_id == AV_CODEC_ID_VC1) ? sizeof(start_code) : 0) > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->dwSliceDataLocation = mapped_ptr - mapped_data; if (i < ctx_pic->slice_count - 1) slice->wNumberMBsInSlice = slice[1].wNumberMBsInSlice - slice[0].wNumberMBsInSlice; @@ -129,11 +137,13 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU mapped_ptr[3] = 0x0b; mapped_ptr += sizeof(start_code); + bitstream_size -= sizeof(start_code); slice->dwSliceBitsInBuffer += sizeof(start_code) * 8; } memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 3abec89a4535fd564a179be8387ea0df6a4f4ab0 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:32 +0200 Subject: [PATCH 7/7] avcodec/d3d12va_vp9: check size of frame bitstream against available buffer size (cherry picked from commit a633df56a455ffb75cd97af784cb1f2d2ec2bca9) --- libavcodec/d3d12va_vp9.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_vp9.c b/libavcodec/d3d12va_vp9.c index 6f1f933fdd..f224044279 100644 --- a/libavcodec/d3d12va_vp9.c +++ b/libavcodec/d3d12va_vp9.c @@ -88,12 +88,18 @@ static int d3d12va_vp9_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VP9SharedContext *h = avctx->priv_data; VP9DecodePictureContext *ctx_pic = h->frames[CUR_FRAME].hwaccel_picture_private; void *mapped_data; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + if (ctx_pic->slice.SliceBytesInBuffer > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
