PR #23742 opened by Timo Rothenpieler (BtbN) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23742 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23742.patch
D3D12VA fully trusts the return value of ff_d3d12va_get_suitable_max_bitstream_size(), which just returns av_image_get_buffer_size(frames_ctx->sw_format, avctx->coded_width, avctx->coded_height, 1). No size checking was done. So a file specifically crafted to bust that limit is able to overflow that buffer. At least I wasn't able to find any size checking done anywhere, so this is what this series adds. This was originally found by depthfirst, but only the instance the first commit fixes. When I looked into it more, I found the same issue in all other d3d12va decoders, as well as the other instance inside of the AV1 one. >From 8ec7a2f63b539fdf574732ec65f706ec9bd5afcd Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Wed, 8 Jul 2026 15:19:33 +0200 Subject: [PATCH 1/7] avcodec/d3d12va_av1: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_av1.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index fdc7195d12..a3ddf0495a 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -112,6 +112,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const AV1DecContext *h = avctx->priv_data; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; void *mapped_data; @@ -121,6 +122,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU args->Size = sizeof(DXVA_Tile_AV1) * ctx_pic->tile_count; args->pData = ctx_pic->tiles; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + input_args->CompressedBitstream = (D3D12_VIDEO_DECODE_COMPRESSED_BITSTREAM){ .pBuffer = buffer, .Offset = 0, -- 2.52.0 >From 3731d9d28a9503664e249f74e7323d0e1ccdbcd0 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:29:01 +0200 Subject: [PATCH 2/7] avcodec/d3d12va_av1: check slice bitstream size against available buffer size --- libavcodec/d3d12va_av1.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libavcodec/d3d12va_av1.c b/libavcodec/d3d12va_av1.c index a3ddf0495a..d4c511d5e6 100644 --- a/libavcodec/d3d12va_av1.c +++ b/libavcodec/d3d12va_av1.c @@ -33,6 +33,7 @@ typedef struct D3D12AV1DecodeContext { D3D12VADecodeContext ctx; uint8_t *bitstream_buffer; + size_t bitstream_size; } D3D12AV1DecodeContext; #define D3D12_AV1_DECODE_CONTEXT(avctx) ((D3D12AV1DecodeContext *)D3D12VA_DECODE_CONTEXT(avctx)) @@ -77,6 +78,7 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, const AV1DecContext *h = avctx->priv_data; const AV1RawFrameHeader *frame_header = h->raw_frame_header; AV1DecodePictureContext *ctx_pic = h->cur_frame.hwaccel_picture_private; + D3D12AV1DecodeContext *av1_ctx = D3D12_AV1_DECODE_CONTEXT(avctx); int offset = 0; uint32_t tg_start, tg_end; @@ -91,7 +93,11 @@ static int d3d12va_av1_decode_slice(AVCodecContext *avctx, ctx_pic->bitstream = (uint8_t *)buffer; ctx_pic->bitstream_size = size; } else { - ctx_pic->bitstream = D3D12_AV1_DECODE_CONTEXT(avctx)->bitstream_buffer; + if (av1_ctx->bitstream_size < ctx_pic->bitstream_size + size) { + av_log(avctx, AV_LOG_ERROR, "Slice bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + ctx_pic->bitstream = av1_ctx->bitstream_buffer; memcpy(ctx_pic->bitstream + ctx_pic->bitstream_size, buffer, size); tg_start = h->tg_start; tg_end = h->tg_end; @@ -180,7 +186,8 @@ static av_cold int d3d12va_av1_decode_init(AVCodecContext *avctx) return ret; if (!av1_ctx->bitstream_buffer) { - av1_ctx->bitstream_buffer = av_malloc(ff_d3d12va_get_suitable_max_bitstream_size(avctx)); + av1_ctx->bitstream_size = ff_d3d12va_get_suitable_max_bitstream_size(avctx); + av1_ctx->bitstream_buffer = av_malloc(av1_ctx->bitstream_size); if (!av1_ctx->bitstream_buffer) return AVERROR(ENOMEM); } @@ -194,6 +201,7 @@ static av_cold int d3d12va_av1_decode_uninit(AVCodecContext *avctx) if (ctx->bitstream_buffer) av_freep(&ctx->bitstream_buffer); + ctx->bitstream_size = 0; return ff_d3d12va_decode_uninit(avctx); } -- 2.52.0 >From a2cae910ac68540319d4306ccb33e445c15a301e Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:38:37 +0200 Subject: [PATCH 3/7] avcodec/d3d12va_h264: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_h264.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_h264.c b/libavcodec/d3d12va_h264.c index dec9344aad..4997b4f583 100644 --- a/libavcodec/d3d12va_h264.c +++ b/libavcodec/d3d12va_h264.c @@ -105,6 +105,7 @@ static int d3d12va_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const H264Context *h = avctx->priv_data; const H264Picture *current_picture = h->cur_pic_ptr; H264DecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -113,6 +114,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_H264_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -127,14 +129,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 1158c16cdc47a3ce4d881bdb8441e2e68e39253b Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:40:32 +0200 Subject: [PATCH 4/7] avcodec/d3d12va_hevc: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_hevc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_hevc.c b/libavcodec/d3d12va_hevc.c index e72d49b7d9..108cf34c05 100644 --- a/libavcodec/d3d12va_hevc.c +++ b/libavcodec/d3d12va_hevc.c @@ -101,6 +101,7 @@ static int d3d12va_hevc_decode_slice(AVCodecContext *avctx, const uint8_t *buffe #define START_CODE_SIZE 3 static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const HEVCContext *h = avctx->priv_data; const HEVCFrame *current_picture = h->cur_frame; HEVCDecodePictureContext *ctx_pic = current_picture->hwaccel_picture_private; @@ -109,6 +110,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU uint8_t *mapped_data, *mapped_ptr; DXVA_Slice_HEVC_Short *slice; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + UINT bitstream_size = ctx->bitstream_size; if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, (void **)&mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); @@ -123,14 +125,22 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU position = slice->BSNALunitDataLocation; size = slice->SliceBytesInBuffer; + if (START_CODE_SIZE + size > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->SliceBytesInBuffer += START_CODE_SIZE; slice->BSNALunitDataLocation = mapped_ptr - mapped_data; *(uint32_t *)mapped_ptr = START_CODE; mapped_ptr += START_CODE_SIZE; + bitstream_size -= START_CODE_SIZE; memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 108b5e1b0060ea05523b6d33bf3b385309af7aa7 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:42:38 +0200 Subject: [PATCH 5/7] avcodec/d3d12va_mpeg2: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_mpeg2.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_mpeg2.c b/libavcodec/d3d12va_mpeg2.c index 47e453dd5e..de9f0c71d1 100644 --- a/libavcodec/d3d12va_mpeg2.c +++ b/libavcodec/d3d12va_mpeg2.c @@ -90,6 +90,7 @@ static int d3d12va_mpeg2_decode_slice(AVCodecContext *avctx, const uint8_t *buff static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const MpegEncContext *s = avctx->priv_data; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -105,6 +106,11 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU .End = ctx_pic->bitstream_size, }; + if (ctx_pic->bitstream_size > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, &range, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 >From 9ca229083e1c04bf3a9e8ed14cd4033164b0c0ea Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:20 +0200 Subject: [PATCH 6/7] avcodec/d3d12va_vc1: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_vc1.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libavcodec/d3d12va_vc1.c b/libavcodec/d3d12va_vc1.c index e64a8e0630..b6d2120957 100644 --- a/libavcodec/d3d12va_vc1.c +++ b/libavcodec/d3d12va_vc1.c @@ -94,6 +94,7 @@ static int d3d12va_vc1_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VC1Context *v = avctx->priv_data; const MpegEncContext *s = &v->s; D3D12DecodePictureContext *ctx_pic = s->cur_pic.ptr->hwaccel_picture_private; @@ -101,6 +102,7 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU const unsigned mb_count = s->mb_width * (s->mb_height >> v->field_mode); uint8_t *mapped_data, *mapped_ptr; + UINT bitstream_size = ctx->bitstream_size; static const uint8_t start_code[] = { 0, 0, 1, 0x0d }; @@ -115,6 +117,12 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU unsigned position = slice->dwSliceDataLocation; unsigned size = slice->dwSliceBitsInBuffer / 8; + if (size + ((avctx->codec_id == AV_CODEC_ID_VC1) ? sizeof(start_code) : 0) > bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + ID3D12Resource_Unmap(buffer, 0, NULL); + return AVERROR(EINVAL); + } + slice->dwSliceDataLocation = mapped_ptr - mapped_data; if (i < ctx_pic->slice_count - 1) slice->wNumberMBsInSlice = slice[1].wNumberMBsInSlice - slice[0].wNumberMBsInSlice; @@ -129,11 +137,13 @@ static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPU mapped_ptr[3] = 0x0b; mapped_ptr += sizeof(start_code); + bitstream_size -= sizeof(start_code); slice->dwSliceBitsInBuffer += sizeof(start_code) * 8; } memcpy(mapped_ptr, &ctx_pic->bitstream[position], size); mapped_ptr += size; + bitstream_size -= size; } ID3D12Resource_Unmap(buffer, 0, NULL); -- 2.52.0 >From 75f93c7b47f196ad6ddda33bcda5e74a5021e5d8 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Thu, 9 Jul 2026 00:50:32 +0200 Subject: [PATCH 7/7] avcodec/d3d12va_vp9: check size of frame bitstream against available buffer size --- libavcodec/d3d12va_vp9.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/d3d12va_vp9.c b/libavcodec/d3d12va_vp9.c index 6f1f933fdd..f224044279 100644 --- a/libavcodec/d3d12va_vp9.c +++ b/libavcodec/d3d12va_vp9.c @@ -88,12 +88,18 @@ static int d3d12va_vp9_decode_slice(AVCodecContext *avctx, const uint8_t *buffer static int update_input_arguments(AVCodecContext *avctx, D3D12_VIDEO_DECODE_INPUT_STREAM_ARGUMENTS *input_args, ID3D12Resource *buffer) { + D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx); const VP9SharedContext *h = avctx->priv_data; VP9DecodePictureContext *ctx_pic = h->frames[CUR_FRAME].hwaccel_picture_private; void *mapped_data; D3D12_VIDEO_DECODE_FRAME_ARGUMENT *args; + if (ctx_pic->slice.SliceBytesInBuffer > ctx->bitstream_size) { + av_log(avctx, AV_LOG_ERROR, "Input frame bitstream size exceeds internal buffer!\n"); + return AVERROR(EINVAL); + } + if (FAILED(ID3D12Resource_Map(buffer, 0, NULL, &mapped_data))) { av_log(avctx, AV_LOG_ERROR, "Failed to map D3D12 Buffer resource!\n"); return AVERROR(EINVAL); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
