PR #23856 opened by ffmpeg-devel URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23856 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23856.patch
**Backport:** https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23742 D3D12VA fully trusts the return value of ff_d3d12va_get_suitable_max_bitstream_size(), which just returns av_image_get_buffer_size(frames_ctx->sw_format, avctx->coded_width, avctx->coded_height, 1). No size checking was done. So a file specifically crafted to bust that limit is able to overflow that buffer. At least I wasn't able to find any size checking done anywhere, so this is what this series adds. This was originally found by depthfirst, but only the instance the first commit fixes. When I looked into it more, I found the same issue in all other d3d12va decoders, as well as the other instance inside of the AV1 one. >From 969c77b5549538e60e6f94eebcaa5952bb2ee061 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Wed, 8 Jul 2026 15:19:33 +0200 Subject: [PATCH 1/7] avcodec/d3d12va_av1: check size of frame bitstream against available buffer size (cherry picked from commit e1ef95c779a3757d21ab970c06aac3e9e1b71974) --- libavcodec/d3d12va_av1.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index 4a4d207b4f..77cb06bfdd 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -109,6 +109,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const AV1DecContext *h = avctx->priv_data; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; void *mapped_data; @@ -118,6 +119,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU args->Size = sizeof(DXVA_Tile_AV1) * ctx_pic->tile_count; args->pData = ctx_pic->tiles; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + input_args->CompressedBitstream = (D3D12_VIDEO_DECODE_COMPRESSED_BITSTREAM){ .pBuffer = buffer, .Offset = 0, -- 2.52.0 >From 2f9d65884fcfb31772ee3dd4767e63e22a4807ca Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:29:01 +0200 Subject: [PATCH 2/7] avcodec/d3d12va_av1: check slice bitstream size against available buffer size (cherry picked from commit 7309ad9fd78d875e60427daa52260834e27ef2ea) --- libavcodec/d3d12va_av1.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index 77cb06bfdd..91256c9f28 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -33,6 +33,7 @@ typedef struct D3D12AV1DecodeContext { D3D12VADecodeContext ctx; uint8_t *bitstream_buffer; + size_t bitstream_size; } D3D12AV1DecodeContext; #define D3D12_AV1_DECODE_CONTEXT(avctx) ((D3D12AV1DecodeContext *)D3D12VA_DECODE_CONTEXT(avctx)) @@ -74,6 +75,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, const AV1DecContext *h = avctx->priv_data; const AV1RawFrameHeader *frame_header = h->raw_frame_header; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; + D3D12AV1DecodeContext *av1_ctx = D3D12_AV1_DECODE_CONTEXT(avctx); int offset = 0; uint32_t tg_start, tg_end; @@ -88,7 +90,11 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, ctx_pic->bitstream = (uint8_t *)buffer; ctx_pic->bitstream_size = size; } else { - ctx_pic->bitstream = D3D12_AV1_DECODE_CONTEXT(avctx)->bitstream_buffer; + if (ctx_pic->bitstream_size + (uint64_t)size > av1_ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Slice bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + ctx_pic->bitstream = av1_ctx->bitstream_buffer; memcpy(ctx_pic->bitstream + ctx_pic->bitstream_size, buffer, size); tg_start = h->tg_start; tg_end = h->tg_end; @@ -177,7 +183,8 @@ static int d3d12va_av1_decode_init(AVCodecContext *avctx) return ret; if (!av1_ctx->bitstream_buffer) { - av1_ctx->bitstream_buffer = av_malloc(ff_d3d12va_get_suitable_max_bitstream_size(avctx)); + av1_ctx->bitstream_size = ff_d3d12va_get_suitable_max_bitstream_size(avctx); + av1_ctx->bitstream_buffer = av_malloc(av1_ctx->bitstream_size); if (!av1_ctx->bitstream_buffer) return AVERROR(ENOMEM); } @@ -191,6 +198,7 @@ static int d3d12va_av1_decode_uninit(AVCodecContext *avctx) if (ctx->bitstream_buffer) av_freep(&ctx->bitstream_buffer); + ctx->bitstream_size = 0; return ff_d3d12va_decode_uninit(avctx); } -- 2.52.0 >From f9e4940772182b32de41e55db87a23a740f0284f Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:38:37 +0200 Subject: [PATCH 3/7] avcodec/d3d12va_h264: check size of frame bitstream against available buffer size (cherry picked from commit 19e81f9260e64c43604556cbd4079fadee4ac4f1) --- libavcodec/d3d12va_h264.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_h264.c b/libavcodec/d3d12va_h264.c index b2fe2955c8..caed3058d6 100644 --- a/libavcodec/d3d12va_h264.c +++ b/libavcodec/d3d12va_h264.c @@ -104,6 +104,7 @@ static int d3d12va_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const H264Context *h = avctx->priv_data; const H264Picture *current_picture = h->cur_pic_ptr; H264DecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -112,6 +113,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_H264_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -126,14 +128,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + (uint64_t)size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From c2661d570f2c361f26db6de872b28b8a87708b00 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:40:32 +0200 Subject: [PATCH 4/7] avcodec/d3d12va_hevc: check size of frame bitstream against available buffer size (cherry picked from commit fea3f13a1cc5395b0ab6db95dd20e7cf6023c311) --- libavcodec/d3d12va_hevc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_hevc.c b/libavcodec/d3d12va_hevc.c index 7686f0eb6c..6d956dab26 100644 --- a/libavcodec/d3d12va_hevc.c +++ b/libavcodec/d3d12va_hevc.c @@ -98,6 +98,7 @@ static int d3d12va_hevc_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const HEVCContext *h = avctx->priv_data; const HEVCFrame *current_picture = h->cur_frame; HEVCDecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -106,6 +107,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_HEVC_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -120,14 +122,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + (uint64_t)size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 6b26c4abd2610a83975be92d32cf9d08c2a7984d Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:42:38 +0200 Subject: [PATCH 5/7] avcodec/d3d12va_mpeg2: check size of frame bitstream against available buffer size (cherry picked from commit 26a9f9b3aeb054901b69518223af0d6f4f991fd9) --- libavcodec/d3d12va_mpeg2.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_mpeg2.c b/libavcodec/d3d12va_mpeg2.c index 86a7d97b34..44f15dc8d4 100644 --- a/libavcodec/d3d12va_mpeg2.c +++ b/libavcodec/d3d12va_mpeg2.c @@ -87,6 +87,7 @@ static int d3d12va_mpeg2_decode_slice(AVCodecContext *avctx, const uint8_t *buff static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const MpegEncContext *s = avctx->priv_data; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -102,6 +103,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU .End = ctx_pic->bitstream_size, }; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, &range, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 >From 60213cca34d00fdea764463abc5154877c8c4a3b Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:20 +0200 Subject: [PATCH 6/7] avcodec/d3d12va_vc1: check size of frame bitstream against available buffer size (cherry picked from commit 1307db3d3c95223b1d2164844e0542cddbb2a401) --- libavcodec/d3d12va_vc1.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_vc1.c b/libavcodec/d3d12va_vc1.c index dccc0fbffa..4dfeaa824e 100644 --- a/libavcodec/d3d12va_vc1.c +++ b/libavcodec/d3d12va_vc1.c @@ -91,6 +91,7 @@ static int d3d12va_vc1_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VC1Context *v = avctx->priv_data; const MpegEncContext *s = &v->s; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -98,6 +99,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU const unsigned mb_count = s->mb_width * (s->mb_height >> v->field_mode); uint8_t *mapped_data, *mapped_ptr; + UINT bitstream_size = ctx->bitstream_size; static const uint8_t start_code[] = { 0, 0, 1, 0x0d }; @@ -112,6 +114,12 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU unsigned position = slice->dwSliceDataLocation; unsigned size = slice->dwSliceBitsInBuffer / 8; + if ((uint64_t)size + ((avctx->codec_id == AV_CODEC_ID_VC1) ? sizeof(start_code) : 0) > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->dwSliceDataLocation = mapped_ptr - mapped_data; if (i < ctx_pic->slice_count - 1) slice->wNumberMBsInSlice = slice[1].wNumberMBsInSlice - slice[0].wNumberMBsInSlice; @@ -126,11 +134,13 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU mapped_ptr[3] = 0x0b; mapped_ptr += sizeof(start_code); + bitstream_size -= sizeof(start_code); slice->dwSliceBitsInBuffer += sizeof(start_code) * 8; } memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From b9a218bc1eb0d5ebb56aa9dcd8e4525be495f7f7 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:32 +0200 Subject: [PATCH 7/7] avcodec/d3d12va_vp9: check size of frame bitstream against available buffer size (cherry picked from commit a633df56a455ffb75cd97af784cb1f2d2ec2bca9) --- libavcodec/d3d12va_vp9.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_vp9.c b/libavcodec/d3d12va_vp9.c index 3476768e61..7dd609cc0e 100644 --- a/libavcodec/d3d12va_vp9.c +++ b/libavcodec/d3d12va_vp9.c @@ -85,12 +85,18 @@ static int d3d12va_vp9_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VP9SharedContext *h = avctx->priv_data; VP9DecodePictureContext *ctx_pic = h->frames[CUR_FRAME].hwaccel_picture_private; void *mapped_data; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + if (ctx_pic->slice.SliceBytesInBuffer > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
