PR #22265 opened by Lynne URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22265 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22265.patch
This PR implements preliminary support for Vulkan. It adds a new API to attach a hardware device (not specific to Vulkan, lets support for other APIs too). It initializes a SPIR-V compiler. And finally, it creates a hardware frames context (and frame pool) to output Vulkan frames. The code is only active and compiled if CONFIG_UNSTABLE has been set. The plan for SPIR-V is to use a new SPIR-V assembler I wrote: https://code.ffmpeg.org/Lynne/spindle libglslang/libshaderc will only be used as a fallback during development/debugging and will be removed once the new SPIR-V assembler is used. >From ab754c5e8b9344dc59bed78fee6448680f506784 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sat, 7 Feb 2026 21:51:44 +0100 Subject: [PATCH 1/5] swscale: add a function to attach a Vulkan device Necessary for filtering. Sponsored-by: Sovereign Tech Fund --- libswscale/Makefile | 2 ++ libswscale/swscale.c | 1 + libswscale/swscale.h | 6 ++++++ libswscale/swscale_internal.h | 8 ++++++++ libswscale/utils.c | 18 ++++++++++++++++++ libswscale/vulkan.c | 19 +++++++++++++++++++ 6 files changed, 54 insertions(+) create mode 100644 libswscale/vulkan.c diff --git a/libswscale/Makefile b/libswscale/Makefile index bde1144897..dc4323fef5 100644 --- a/libswscale/Makefile +++ b/libswscale/Makefile @@ -33,6 +33,8 @@ OBJS-$(CONFIG_UNSTABLE) += \ ops_memcpy.o \ ops_optimizer.o \ +OBJS-$(CONFIG_VULKAN) += vulkan.o + # Objects duplicated from other libraries for shared builds SHLIBOBJS += log2_tab.o half2float.o diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 219382505c..77d935572a 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -34,6 +34,7 @@ #include "config.h" #include "swscale_internal.h" #include "swscale.h" +#include "libavutil/hwcontext_vulkan.h" DECLARE_ALIGNED(8, const uint8_t, ff_dither_8x8_128)[9][8] = { { 36, 68, 60, 92, 34, 66, 58, 90, }, diff --git a/libswscale/swscale.h b/libswscale/swscale.h index 495d500f14..0847480d7f 100644 --- a/libswscale/swscale.h +++ b/libswscale/swscale.h @@ -262,6 +262,12 @@ SwsContext *sws_alloc_context(void); */ void sws_free_context(SwsContext **ctx); +/** + * Attach a hardware device to the context. + * Required to be called if either the input or output frames are hardware. + */ +int sws_context_attach_hwcontext(SwsContext *ctx, AVBufferRef *hwctx); + /*************************** * Supported frame formats * ***************************/ diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 5c58272664..d5ec961d60 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -37,6 +37,8 @@ #include "libavutil/pixfmt.h" #include "libavutil/pixdesc.h" #include "libavutil/slicethread.h" +#include "libavutil/vulkan.h" + #if HAVE_ALTIVEC #include "libavutil/ppc/util_altivec.h" #endif @@ -326,6 +328,10 @@ typedef void (*planarX2_YV12_fn)(uint8_t *dst, uint8_t *dst2, struct SwsSlice; struct SwsFilterDescriptor; +typedef struct SwsInternalVulkan { + FFVulkanContext ctx; +} SwsInternalVulkan; + /* This struct should be aligned on at least a 32-byte boundary. */ struct SwsInternal { /* Currently active user-facing options. Also contains AVClass */ @@ -701,6 +707,8 @@ struct SwsInternal { int color_conversion_warned; Half2FloatTables *h2f_tables; + + SwsInternalVulkan vk; }; //FIXME check init (where 0) diff --git a/libswscale/utils.c b/libswscale/utils.c index 52095ab2c7..b710af5891 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -57,6 +57,7 @@ #include "libavutil/x86/asm.h" #include "libavutil/x86/cpu.h" #include "libavutil/loongarch/cpu.h" +#include "config_components.h" #include "rgb2rgb.h" #include "swscale.h" @@ -2252,6 +2253,20 @@ fail: return NULL; } +int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx) +{ +#if CONFIG_VULKAN + SwsInternal *c = sws_internal(sws); + int err = ff_vk_init(&c->vk.ctx, sws, hwctx, NULL); + if (err < 0) + return err; + + return 0; +#else + return AVERROR(ENOTSUP); +#endif +} + void sws_freeContext(SwsContext *sws) { SwsInternal *c = sws_internal(sws); @@ -2330,6 +2345,9 @@ void sws_freeContext(SwsContext *sws) av_freep(&c->xyz_scratch); ff_free_filters(c); +#if CONFIG_VULKAN + ff_vk_uninit(&c->vk.ctx); +#endif av_free(c); } diff --git a/libswscale/vulkan.c b/libswscale/vulkan.c new file mode 100644 index 0000000000..fc8a1fa47b --- /dev/null +++ b/libswscale/vulkan.c @@ -0,0 +1,19 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/vulkan.c" -- 2.52.0 >From f6bb5c18e8db74c1b4a0b8b4b45114113492ae89 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sun, 8 Feb 2026 11:21:52 +0100 Subject: [PATCH 2/5] swscale: initialize a GLSL compilation context, if supported Sponsored-by: Sovereign Tech Fund --- libavutil/vulkan_spirv.h | 1 + libswscale/Makefile | 2 ++ libswscale/swscale_internal.h | 2 ++ libswscale/utils.c | 18 ++++++++++++++++++ libswscale/vulkan_glslang.c | 19 +++++++++++++++++++ libswscale/vulkan_shaderc.c | 19 +++++++++++++++++++ 6 files changed, 61 insertions(+) create mode 100644 libswscale/vulkan_glslang.c create mode 100644 libswscale/vulkan_shaderc.c diff --git a/libavutil/vulkan_spirv.h b/libavutil/vulkan_spirv.h index c13b50f8e7..0bdcf8962f 100644 --- a/libavutil/vulkan_spirv.h +++ b/libavutil/vulkan_spirv.h @@ -22,6 +22,7 @@ #include "vulkan.h" #include "config.h" +#include "config_components.h" typedef struct FFVkSPIRVCompiler { void *priv; diff --git a/libswscale/Makefile b/libswscale/Makefile index dc4323fef5..217d8e4e14 100644 --- a/libswscale/Makefile +++ b/libswscale/Makefile @@ -34,6 +34,8 @@ OBJS-$(CONFIG_UNSTABLE) += \ ops_optimizer.o \ OBJS-$(CONFIG_VULKAN) += vulkan.o +OBJS-$(CONFIG_LIBGLSLANG) += vulkan_glslang.o +OBJS-$(CONFIG_LIBSHADERC) += vulkan_shaderc.o # Objects duplicated from other libraries for shared builds SHLIBOBJS += log2_tab.o half2float.o diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index d5ec961d60..94e6f21959 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -38,6 +38,7 @@ #include "libavutil/pixdesc.h" #include "libavutil/slicethread.h" #include "libavutil/vulkan.h" +#include "libavutil/vulkan_spirv.h" #if HAVE_ALTIVEC #include "libavutil/ppc/util_altivec.h" @@ -330,6 +331,7 @@ struct SwsFilterDescriptor; typedef struct SwsInternalVulkan { FFVulkanContext ctx; + FFVkSPIRVCompiler *spvc; } SwsInternalVulkan; /* This struct should be aligned on at least a 32-byte boundary. */ diff --git a/libswscale/utils.c b/libswscale/utils.c index b710af5891..5716a0fb7c 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -2261,6 +2261,20 @@ int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx) if (err < 0) return err; +#if CONFIG_LIBSHADERC + c->vk.spvc = ff_vk_shaderc_init(); + if (!c->vk.spvc) { + ff_vk_uninit(&c->vk.ctx); + return AVERROR(ENOMEM); + } +#elif CONFIG_LIBGLSLANG + c->vk.spvc = ff_vk_glslang_init(); + if (!c->vk.spvc) { + ff_vk_uninit(&c->vk.ctx); + return AVERROR(ENOMEM); + } +#endif + return 0; #else return AVERROR(ENOTSUP); @@ -2348,6 +2362,10 @@ void sws_freeContext(SwsContext *sws) #if CONFIG_VULKAN ff_vk_uninit(&c->vk.ctx); #endif +#if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG + if (c->vk.spvc) + c->vk.spvc->uninit(&c->vk.spvc); +#endif av_free(c); } diff --git a/libswscale/vulkan_glslang.c b/libswscale/vulkan_glslang.c new file mode 100644 index 0000000000..9aa41567a3 --- /dev/null +++ b/libswscale/vulkan_glslang.c @@ -0,0 +1,19 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/vulkan_glslang.c" diff --git a/libswscale/vulkan_shaderc.c b/libswscale/vulkan_shaderc.c new file mode 100644 index 0000000000..9f60bf4dfd --- /dev/null +++ b/libswscale/vulkan_shaderc.c @@ -0,0 +1,19 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/vulkan_shaderc.c" -- 2.52.0 >From 2e5ca59857b180602139ecb0e41a74b43660a792 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sun, 15 Feb 2026 16:24:21 +0100 Subject: [PATCH 3/5] lavu/vulkan: don't redefine RET, if already defined RET is a very common name for a define that we use in some other parts of our codebase. Sponsored-by: Sovereign Tech Fund --- libavutil/vulkan.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavutil/vulkan.h b/libavutil/vulkan.h index 9d1a53e2be..806c5c879d 100644 --- a/libavutil/vulkan.h +++ b/libavutil/vulkan.h @@ -65,11 +65,13 @@ } while (0) /* Helper, pretty much every Vulkan return value needs to be checked */ +#ifndef RET #define RET(x) \ do { \ if ((err = (x)) < 0) \ goto fail; \ } while (0) +#endif /* Convenience macros for specialization lists */ #define SPEC_LIST_MAX 256 -- 2.52.0 >From 8df59b043ad1ac0a7eea4bffc02a1cc4e84eee7c Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Sun, 15 Feb 2026 17:59:04 +0100 Subject: [PATCH 4/5] swscale: add support for outputting Vulkan frames Sponsored-by: Sovereign Tech Fund --- libswscale/swscale.c | 47 ++++++++++++++++++++++++++++++++--- libswscale/swscale_internal.h | 2 ++ libswscale/utils.c | 3 +++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 77d935572a..0448e9654f 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -1210,6 +1210,32 @@ void sws_frame_end(SwsContext *sws) c->src_ranges.nb_ranges = 0; } +static int sws_hwframe_ref_init(SwsContext *sws) +{ + int err; + SwsInternal *c = sws_internal(sws); + + av_buffer_unref(&c->vk.hwframe_ref); + c->vk.hwframe_ref = av_hwframe_ctx_alloc(c->vk.ctx.device_ref); + if (!c->vk.hwframe_ref) + return AVERROR(ENOMEM); + + AVHWFramesContext *hwfc = (AVHWFramesContext *)c->vk.hwframe_ref->data; + hwfc->format = AV_PIX_FMT_VULKAN; + hwfc->sw_format = sws->dst_format; + hwfc->width = sws->dst_w; + hwfc->height = sws->dst_h; + + err = av_hwframe_ctx_init(c->vk.hwframe_ref); + if (err) { + av_log(sws, AV_LOG_ERROR, "AVHWFramesContext init failed: %s!\n", + av_err2str(err)); + return err; + } + + return 0; +} + int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src) { SwsInternal *c = sws_internal(sws); @@ -1222,11 +1248,24 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src) if (!dst->buf[0]) { dst->width = sws->dst_w; dst->height = sws->dst_h; - dst->format = sws->dst_format; - ret = av_frame_get_buffer(dst, 0); - if (ret < 0) - return ret; + if (c->vk.initialized) { + dst->format = AV_PIX_FMT_VULKAN; + if (!c->vk.hwframe_ref) { + ret = sws_hwframe_ref_init(sws); + if (ret < 0) + return ret; + } + ret = av_hwframe_get_buffer(c->vk.hwframe_ref, dst, 0); + if (ret < 0) + return ret; + } else { + dst->format = sws->dst_format; + ret = av_frame_get_buffer(dst, 0); + if (ret < 0) + return ret; + } + allocated = 1; } diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 94e6f21959..5af48b5149 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -332,6 +332,8 @@ struct SwsFilterDescriptor; typedef struct SwsInternalVulkan { FFVulkanContext ctx; FFVkSPIRVCompiler *spvc; + AVBufferRef *hwframe_ref; + int initialized; } SwsInternalVulkan; /* This struct should be aligned on at least a 32-byte boundary. */ diff --git a/libswscale/utils.c b/libswscale/utils.c index 5716a0fb7c..fa184dbdbb 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -2275,6 +2275,8 @@ int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx) } #endif + c->vk.initialized = 1; + return 0; #else return AVERROR(ENOTSUP); @@ -2360,6 +2362,7 @@ void sws_freeContext(SwsContext *sws) ff_free_filters(c); #if CONFIG_VULKAN + av_buffer_unref(&c->vk.hwframe_ref); ff_vk_uninit(&c->vk.ctx); #endif #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG -- 2.52.0 >From 7ee58801be5644de8a21294b4df7e0a63d10a2c6 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Mon, 23 Feb 2026 15:24:26 +0100 Subject: [PATCH 5/5] swscale: only enable the Vulkan code if CONFIG_UNSTABLE is set Sponsored-by: Sovereign Tech Fund --- libswscale/swscale.c | 7 ++++++- libswscale/swscale_internal.h | 4 ++++ libswscale/utils.c | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 0448e9654f..820520e839 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -1210,6 +1210,7 @@ void sws_frame_end(SwsContext *sws) c->src_ranges.nb_ranges = 0; } +#if CONFIG_UNSTABLE static int sws_hwframe_ref_init(SwsContext *sws) { int err; @@ -1235,6 +1236,7 @@ static int sws_hwframe_ref_init(SwsContext *sws) return 0; } +#endif int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src) { @@ -1249,6 +1251,7 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src) dst->width = sws->dst_w; dst->height = sws->dst_h; +#if CONFIG_UNSTABLE if (c->vk.initialized) { dst->format = AV_PIX_FMT_VULKAN; if (!c->vk.hwframe_ref) { @@ -1259,7 +1262,9 @@ int sws_frame_start(SwsContext *sws, AVFrame *dst, const AVFrame *src) ret = av_hwframe_get_buffer(c->vk.hwframe_ref, dst, 0); if (ret < 0) return ret; - } else { + } else +#endif + { dst->format = sws->dst_format; ret = av_frame_get_buffer(dst, 0); if (ret < 0) diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 5af48b5149..a671b5d3fe 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -329,12 +329,14 @@ typedef void (*planarX2_YV12_fn)(uint8_t *dst, uint8_t *dst2, struct SwsSlice; struct SwsFilterDescriptor; +#if CONFIG_UNSTABLE typedef struct SwsInternalVulkan { FFVulkanContext ctx; FFVkSPIRVCompiler *spvc; AVBufferRef *hwframe_ref; int initialized; } SwsInternalVulkan; +#endif /* This struct should be aligned on at least a 32-byte boundary. */ struct SwsInternal { @@ -712,7 +714,9 @@ struct SwsInternal { Half2FloatTables *h2f_tables; +#if CONFIG_UNSTABLE SwsInternalVulkan vk; +#endif }; //FIXME check init (where 0) diff --git a/libswscale/utils.c b/libswscale/utils.c index fa184dbdbb..7b518596f3 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -2255,7 +2255,7 @@ fail: int sws_context_attach_hwcontext(SwsContext *sws, AVBufferRef *hwctx) { -#if CONFIG_VULKAN +#if CONFIG_UNSTABLE && CONFIG_VULKAN SwsInternal *c = sws_internal(sws); int err = ff_vk_init(&c->vk.ctx, sws, hwctx, NULL); if (err < 0) @@ -2361,6 +2361,7 @@ void sws_freeContext(SwsContext *sws) av_freep(&c->xyz_scratch); ff_free_filters(c); +#if CONFIG_UNSTABLE #if CONFIG_VULKAN av_buffer_unref(&c->vk.hwframe_ref); ff_vk_uninit(&c->vk.ctx); @@ -2368,6 +2369,7 @@ void sws_freeContext(SwsContext *sws) #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG if (c->vk.spvc) c->vk.spvc->uninit(&c->vk.spvc); +#endif #endif av_free(c); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
