PR #23848 opened by Raja-89 URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23848 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23848.patch
When a CUDA frame with RGB24 sw_format is received, map the GPU device pointer directly into a LibTorch tensor using torch::from_blob() with CUDA device options, bypassing cudaMemcpy entirely. Changes: - Add fill_model_input_cuda() that extracts the CUdeviceptr from the AVHWFramesContext and wraps it in a PyTorch GPU tensor with a no-op deleter (memory owned by FFmpeg AVBuffer ref-counting). - Handle GPU memory alignment padding via custom strides derived from AVFrame linesize. - Permute the mapped NHWC tensor to NCHW (PyTorch convention). - Manage CUDA context explicitly with cuCtxPushCurrent/cuCtxPopCurrent for thread safety in async filter graphs. - Gracefully fall back to CPU path when sw_format is not RGB24 or when tensor creation fails, by downloading the frame via av_hwframe_transfer_data(). - Add AV_PIX_FMT_CUDA to vf_dnn_processing supported formats. - Guard all CUDA code with CONFIG_CUDA. Tested with: - CUDA Fallback Path (Triggered when sw_format is not RGB24): ffmpeg -y -i input.mp4 -vf "format=rgb24,hwupload_cuda,dnn_processing=dnn_backend=torch:model=dummy_model.pt:device=cuda" -frames:v 5 -c:v h264_nvenc output.mp4 - Standard CPU Path (Regression Test): ffmpeg -y -i input.mp4 -vf "dnn_processing=dnn_backend=torch:model=dummy_model.pt" -frames:v 5 -c:v rawvideo -pix_fmt rgb24 output.avi Signed-off-by: Raja Rathour >From 5de88fed210a5e98fcc01555a5171cb346f3a1b2 Mon Sep 17 00:00:00 2001 From: Raja-89 <[email protected]> Date: Sun, 19 Jul 2026 23:06:06 +0530 Subject: [PATCH] avfilter/dnn: implement zero-copy CUDA tensor mapping for Torch backend When a CUDA frame with RGB24 sw_format is received, map the GPU device pointer directly into a LibTorch tensor using torch::from_blob() with CUDA device options, bypassing cudaMemcpy entirely. Changes: - Add fill_model_input_cuda() that extracts the CUdeviceptr from the AVHWFramesContext and wraps it in a PyTorch GPU tensor with a no-op deleter (memory owned by FFmpeg AVBuffer ref-counting). - Handle GPU memory alignment padding via custom strides derived from AVFrame linesize. - Permute the mapped NHWC tensor to NCHW (PyTorch convention). - Manage CUDA context explicitly with cuCtxPushCurrent/cuCtxPopCurrent for thread safety in async filter graphs. - Gracefully fall back to CPU path when sw_format is not RGB24 or when tensor creation fails, by downloading the frame via av_hwframe_transfer_data(). - Add AV_PIX_FMT_CUDA to vf_dnn_processing supported formats. - Guard all CUDA code with CONFIG_CUDA. Signed-off-by: Raja Rathour <[email protected]> --- libavfilter/dnn/dnn_backend_torch.cpp | 137 +++++++++++++++++++++++++- libavfilter/vf_dnn_processing.c | 4 + 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_torch.cpp b/libavfilter/dnn/dnn_backend_torch.cpp index 9ba6d61377..afc6e15943 100644 --- a/libavfilter/dnn/dnn_backend_torch.cpp +++ b/libavfilter/dnn/dnn_backend_torch.cpp @@ -32,6 +32,11 @@ extern "C" { #include "libavutil/opt.h" #include "libavutil/mem.h" #include "libavutil/cpu.h" +#include "libavutil/pixfmt.h" +#if CONFIG_CUDA +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_cuda.h" +#endif #include "queue.h" #include "safe_queue.h" } @@ -159,6 +164,95 @@ static void deleter(void *arg) av_freep(&arg); } +#if CONFIG_CUDA + +static void cuda_tensor_deleter(void *arg) +{ + /* No-op: GPU memory is owned by FFmpeg AVBuffer ref-counting. + * LibTorch must not free it. */ + (void)arg; +} + +/** + * Map a CUDA frame's GPU pointer directly into a LibTorch tensor, + * bypassing any host-device memory copy. + * + * Only RGB24 sw_format is supported. Returns AVERROR(ENOSYS) for + * unsupported formats so the caller can fall back to the CPU path. + */ +static int fill_model_input_cuda(THModel *th_model, THRequestItem *request, + LastLevelTaskItem *lltask) +{ + TaskItem *task = lltask->task; + THInferRequest *infer_request = request->infer_request; + AVFrame *in_frame = task->in_frame; + AVHWFramesContext *hw_frames_ctx; + AVCUDADeviceContext *cuda_dev_ctx; + CUcontext dummy; + DNNData input; + int height = in_frame->height; + int width = in_frame->width; + int channels, ret; + + if (!in_frame->hw_frames_ctx) { + av_log(th_model->ctx, AV_LOG_ERROR, "CUDA frame has no hw_frames_ctx\n"); + return AVERROR(EINVAL); + } + hw_frames_ctx = (AVHWFramesContext *)in_frame->hw_frames_ctx->data; + + if (hw_frames_ctx->sw_format != AV_PIX_FMT_RGB24) { + av_log(th_model->ctx, AV_LOG_VERBOSE, + "Zero-copy CUDA path requires RGB24 sw_format, " + "falling back to CPU path. " + "Insert scale_cuda=format=rgb24 before dnn_processing " + "to enable zero-copy.\n"); + return AVERROR(ENOSYS); + } + + ret = get_input_th(&th_model->model, &input, NULL); + if (ret != 0) + return ret; + channels = input.dims[dnn_get_channel_idx_by_layout(input.layout)]; + + cuda_dev_ctx = (AVCUDADeviceContext *)hw_frames_ctx->device_ctx->hwctx; + if (cuCtxPushCurrent(cuda_dev_ctx->cuda_ctx) != CUDA_SUCCESS) { + av_log(th_model->ctx, AV_LOG_ERROR, "Failed to push CUDA context\n"); + return AVERROR_EXTERNAL; + } + + infer_request->input_tensor = new torch::Tensor(); + infer_request->output = new torch::Tensor(); + + try { + *infer_request->input_tensor = torch::from_blob( + in_frame->data[0], + {1, height, width, channels}, + {(long)(height * in_frame->linesize[0]), + (long)in_frame->linesize[0], + (long)channels, 1L}, + cuda_tensor_deleter, + torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA)); + + /* Permute from NHWC to NCHW (PyTorch convention) */ + *infer_request->input_tensor = + infer_request->input_tensor->permute({0, 3, 1, 2}).contiguous(); + } catch (const c10::Error &e) { + av_log(th_model->ctx, AV_LOG_VERBOSE, + "CUDA zero-copy tensor creation failed (%s), " + "falling back to CPU path\n", e.what()); + th_free_request(infer_request); + cuCtxPopCurrent(&dummy); + return AVERROR(ENOSYS); + } + + request->lltasks[0] = lltask; + request->lltask_count = 1; + + cuCtxPopCurrent(&dummy); + return 0; +} +#endif + static int fill_model_input_th(THModel *th_model, THRequestItem *request) { LastLevelTaskItem *lltask = NULL; @@ -187,6 +281,39 @@ static int fill_model_input_th(THModel *th_model, THRequestItem *request) goto err; } task = lltask->task; + +#if CONFIG_CUDA + if (task->in_frame->format == AV_PIX_FMT_CUDA) { + lltask = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue); + ret = fill_model_input_cuda(th_model, request, lltask); + if (ret != AVERROR(ENOSYS)) { + if (ret < 0) + av_freep(&lltask); + return ret; + } + + /* ENOSYS: unsupported sw_format, fall back to CPU path */ + AVFrame *cpu_frame = av_frame_alloc(); + if (!cpu_frame) { + av_freep(&lltask); + return AVERROR(ENOMEM); + } + ret = av_hwframe_transfer_data(cpu_frame, task->in_frame, 0); + if (ret < 0) { + av_frame_free(&cpu_frame); + av_freep(&lltask); + av_log(th_model->ctx, AV_LOG_ERROR, + "Failed to download CUDA frame to CPU for fallback\n"); + return ret; + } + av_frame_copy_props(cpu_frame, task->in_frame); + av_frame_free(&task->in_frame); + task->in_frame = cpu_frame; + /* Push lltask back so the CPU batching loop can pick it up */ + ff_queue_push_front(th_model->lltask_queue, lltask); + } +#endif + input.dims[height_idx] = task->in_frame->height; input.dims[width_idx] = task->in_frame->width; @@ -480,12 +607,16 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A at::detail::getXPUHooks().initXPU(); #endif } else if (device.is_cuda()) { - // CUDA device - works for both NVIDIA CUDA and AMD ROCm (which uses CUDA-compatible API) +#if CONFIG_CUDA if (!torch::cuda::is_available()) { - av_log(ctx, AV_LOG_ERROR, "CUDA/ROCm is not available\n"); + av_log(ctx, AV_LOG_ERROR, "CUDA is not available\n"); goto fail; } - av_log(ctx, AV_LOG_INFO, "Using CUDA/ROCm device: %s\n", device_name); +#else + av_log(ctx, AV_LOG_ERROR, + "CUDA device requested but FFmpeg was built without CUDA support\n"); + goto fail; +#endif } else if (!device.is_cpu()) { av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name); goto fail; diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c index 7ffa700cc5..6a5c5eb0bc 100644 --- a/libavfilter/vf_dnn_processing.c +++ b/libavfilter/vf_dnn_processing.c @@ -28,6 +28,7 @@ #include "libavutil/avassert.h" #include "libavutil/imgutils.h" #include "filters.h" +#include "formats.h" #include "dnn_filter_common.h" #include "video.h" #include "libswscale/swscale.h" @@ -73,6 +74,7 @@ static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_NV12, + AV_PIX_FMT_CUDA, AV_PIX_FMT_NONE }; @@ -132,6 +134,8 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin return AVERROR(EIO); } return 0; + case AV_PIX_FMT_CUDA: + return 0; default: avpriv_report_missing_feature(ctx, "%s", av_get_pix_fmt_name(fmt)); return AVERROR(EIO); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
