PR #22322 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22322 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22322.patch
AVFrame just really doesn't have the semantics we want. However, there a tangible benefit to having SwsFrame act as a carbon copy of a (subset of) AVFrame. This partially reverts commit 67f36272671be9405248386bfbc467a6a772d5ce. Sponsored-by: Sovereign Tech Fund Signed-off-by: Niklas Haas <[email protected]> Supersedes: #22321 >From fdc83531eceae3f47aca05ad434e64716353b6fd Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 27 Feb 2026 21:53:24 +0100 Subject: [PATCH] swscale/graph: reintroduce SwsFrame AVFrame just really doesn't have the semantics we want. However, there a tangible benefit to having SwsFrame act as a carbon copy of a (subset of) AVFrame. This partially reverts commit 67f36272671be9405248386bfbc467a6a772d5ce. Sponsored-by: Sovereign Tech Fund Signed-off-by: Niklas Haas <[email protected]> --- libswscale/format.c | 12 ++++++ libswscale/format.h | 34 +++++++++++++++++ libswscale/graph.c | 80 +++++++++++++++++---------------------- libswscale/graph.h | 18 ++++----- libswscale/ops.c | 8 ++-- libswscale/ops_internal.h | 5 ++- libswscale/vulkan/ops.c | 4 +- 7 files changed, 97 insertions(+), 64 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index 56f9e53447..3464a8af7c 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -636,6 +636,18 @@ int sws_is_noop(const AVFrame *dst, const AVFrame *src) return 1; } +void ff_sws_frame_from_avframe(SwsFrame *dst, const AVFrame *src) +{ + dst->format = src->format; + dst->width = src->width; + dst->height = src->height; + dst->avframe = src; + for (int i = 0; i < FF_ARRAY_ELEMS(dst->data); i++) { + dst->data[i] = src->data[i]; + dst->linesize[i] = src->linesize[i]; + } +} + #if CONFIG_UNSTABLE /* Returns the type suitable for a pixel after fully decoding/unpacking it */ diff --git a/libswscale/format.h b/libswscale/format.h index 509554492e..d66851893f 100644 --- a/libswscale/format.h +++ b/libswscale/format.h @@ -173,4 +173,38 @@ int ff_sws_encode_colors(SwsContext *ctx, SwsPixelType type, SwsOpList *ops, const SwsFormat *src, const SwsFormat *dst, bool *incomplete); +/** + * Represents a view into a single field of frame data. + * + * Ostensibly, this is a (non-compatible) subset of AVFrame, however, the + * semantics are VERY different. + * + * Unlike AVFrame, this struct does NOT own any data references. All buffers + * referenced by a SwsFrame are managed externally. This merely represents + * a view into data. + * + * This struct is not refcounted, and may be freely copied onto the stack. + */ +typedef struct SwsFrame { + /* Data buffers and line stride */ + uint8_t *data[4]; + int linesize[4]; + + /** + * Dimensions and format + */ + int width, height; + enum AVPixelFormat format; + + /** + * Pointer to the original AVFrame, if there is a 1:1 correspondence. + **/ + const AVFrame *avframe; +} SwsFrame; + +/** + * Initialize a SwsFrame from an AVFrame. + */ +void ff_sws_frame_from_avframe(SwsFrame *dst, const AVFrame *src); + #endif /* SWSCALE_FORMAT_H */ diff --git a/libswscale/graph.c b/libswscale/graph.c index 6a17a12496..304d1f4017 100644 --- a/libswscale/graph.c +++ b/libswscale/graph.c @@ -76,31 +76,32 @@ static int frame_alloc_planes(AVFrame *dst) static int pass_alloc_output(SwsPass *pass) { - if (!pass || pass->output->frame) + if (!pass || pass->output->avframe) return 0; SwsPassBuffer *buffer = pass->output; - AVFrame *frame = av_frame_alloc(); - if (!frame) + AVFrame *avframe = av_frame_alloc(); + if (!avframe) return AVERROR(ENOMEM); - frame->format = pass->format; - frame->width = buffer->width; - frame->height = buffer->height; + avframe->format = pass->format; + avframe->width = buffer->width; + avframe->height = buffer->height; - int ret = frame_alloc_planes(frame); + int ret = frame_alloc_planes(avframe); if (ret < 0) { - av_frame_free(&frame); + av_frame_free(&avframe); return ret; } - buffer->frame = frame; + buffer->avframe = avframe; + ff_sws_frame_from_avframe(&buffer->frame, avframe); return 0; } static void free_buffer(AVRefStructOpaque opaque, void *obj) { SwsPassBuffer *buffer = obj; - av_frame_free(&buffer->frame); + av_frame_free(&buffer->avframe); } SwsPass *ff_sws_graph_add_pass(SwsGraph *graph, enum AVPixelFormat fmt, @@ -163,7 +164,7 @@ static int pass_append(SwsGraph *graph, enum AVPixelFormat fmt, int w, int h, return 0; } -static void frame_shift(const AVFrame *f, const int y, uint8_t *data[4]) +static void frame_shift(const SwsFrame *f, const int y, uint8_t *data[4]) { for (int i = 0; i < 4; i++) { if (f->data[i]) @@ -173,7 +174,7 @@ static void frame_shift(const AVFrame *f, const int y, uint8_t *data[4]) } } -static void run_copy(const AVFrame *out, const AVFrame *in, int y, int h, +static void run_copy(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { uint8_t *in_data[4], *out_data[4]; @@ -199,7 +200,7 @@ static void run_copy(const AVFrame *out, const AVFrame *in, int y, int h, } } -static void run_rgb0(const AVFrame *out, const AVFrame *in, int y, int h, +static void run_rgb0(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { SwsInternal *c = pass->priv; @@ -220,7 +221,7 @@ static void run_rgb0(const AVFrame *out, const AVFrame *in, int y, int h, } } -static void run_xyz2rgb(const AVFrame *out, const AVFrame *in, int y, int h, +static void run_xyz2rgb(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { const SwsInternal *c = pass->priv; @@ -229,7 +230,7 @@ static void run_xyz2rgb(const AVFrame *out, const AVFrame *in, int y, int h, pass->width, h); } -static void run_rgb2xyz(const AVFrame *out, const AVFrame *in, int y, int h, +static void run_rgb2xyz(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { const SwsInternal *c = pass->priv; @@ -250,7 +251,7 @@ static void free_legacy_swscale(void *priv) sws_free_context(&sws); } -static void setup_legacy_swscale(const AVFrame *out, const AVFrame *in, +static void setup_legacy_swscale(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass) { SwsContext *sws = pass->priv; @@ -283,7 +284,7 @@ static inline SwsContext *slice_ctx(const SwsPass *pass, int y) return sws; } -static void run_legacy_unscaled(const AVFrame *out, const AVFrame *in, +static void run_legacy_unscaled(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { SwsContext *sws = slice_ctx(pass, y); @@ -295,7 +296,7 @@ static void run_legacy_unscaled(const AVFrame *out, const AVFrame *in, out->data, out->linesize); } -static void run_legacy_swscale(const AVFrame *out, const AVFrame *in, +static void run_legacy_swscale(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { SwsContext *sws = slice_ctx(pass, y); @@ -632,7 +633,7 @@ static void free_lut3d(void *priv) ff_sws_lut3d_free(&lut); } -static void setup_lut3d(const AVFrame *out, const AVFrame *in, const SwsPass *pass) +static void setup_lut3d(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass) { SwsLut3D *lut = pass->priv; @@ -640,7 +641,7 @@ static void setup_lut3d(const AVFrame *out, const AVFrame *in, const SwsPass *pa ff_sws_lut3d_update(lut, &pass->graph->src.color); } -static void run_lut3d(const AVFrame *out, const AVFrame *in, int y, int h, +static void run_lut3d(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass) { SwsLut3D *lut = pass->priv; @@ -781,14 +782,6 @@ int ff_sws_graph_create(SwsContext *ctx, const SwsFormat *dst, const SwsFormat * graph->field = field; graph->opts_copy = *ctx; - for (int i = 0; i < FF_ARRAY_ELEMS(graph->field_tmp); i++) { - graph->field_tmp[i] = av_frame_alloc(); - if (!graph->field_tmp[i]) { - ret = AVERROR(ENOMEM); - goto error; - } - } - ret = avpriv_slicethread_create(&graph->slicethread, (void *) graph, sws_graph_worker, NULL, ctx->threads); if (ret == AVERROR(ENOSYS)) @@ -827,9 +820,6 @@ void ff_sws_graph_free(SwsGraph **pgraph) } av_free(graph->passes); - for (int i = 0; i < FF_ARRAY_ELEMS(graph->field_tmp); i++) - av_frame_free(&graph->field_tmp[i]); - av_free(graph); *pgraph = NULL; } @@ -875,22 +865,22 @@ void ff_sws_graph_update_metadata(SwsGraph *graph, const SwsColor *color) ff_color_update_dynamic(&graph->src.color, color); } -static const AVFrame *get_field(SwsGraph *graph, const AVFrame *frame, - AVFrame *restrict tmp) +static SwsFrame get_field(SwsGraph *graph, const AVFrame *avframe) { - if (!(frame->flags & AV_FRAME_FLAG_INTERLACED)) { + SwsFrame frame; + ff_sws_frame_from_avframe(&frame, avframe); + + if (!(avframe->flags & AV_FRAME_FLAG_INTERLACED)) { av_assert1(!graph->field); return frame; } - *tmp = *frame; - if (graph->field == FIELD_BOTTOM) { /* Odd rows, offset by one line */ - const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format); + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame.format); for (int i = 0; i < 4; i++) { - if (tmp->data[i]) - tmp->data[i] += frame->linesize[i]; + if (frame.data[i]) + frame.data[i] += frame.linesize[i]; if (desc->flags & AV_PIX_FMT_FLAG_PAL) break; } @@ -898,23 +888,23 @@ static const AVFrame *get_field(SwsGraph *graph, const AVFrame *frame, /* Take only every second line */ for (int i = 0; i < 4; i++) - tmp->linesize[i] <<= 1; + frame.linesize[i] <<= 1; - return tmp; + return frame; } void ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src) { av_assert0(dst->format == graph->dst.hw_format || dst->format == graph->dst.format); av_assert0(src->format == graph->src.hw_format || src->format == graph->src.format); - const AVFrame *src_field = get_field(graph, src, graph->field_tmp[0]); - const AVFrame *dst_field = get_field(graph, dst, graph->field_tmp[1]); + const SwsFrame src_field = get_field(graph, src); + const SwsFrame dst_field = get_field(graph, dst); for (int i = 0; i < graph->num_passes; i++) { const SwsPass *pass = graph->passes[i]; graph->exec.pass = pass; - graph->exec.input = pass->input ? pass->input->output->frame : src_field; - graph->exec.output = pass->output->frame ? pass->output->frame : dst_field; + graph->exec.input = pass->input ? &pass->input->output->frame : &src_field; + graph->exec.output = pass->output->avframe ? &pass->output->frame : &dst_field; if (pass->setup) pass->setup(graph->exec.output, graph->exec.input, pass); avpriv_slicethread_execute(graph->slicethread, pass->num_slices, 0); diff --git a/libswscale/graph.h b/libswscale/graph.h index 9ddf2ad1c9..88d8245218 100644 --- a/libswscale/graph.h +++ b/libswscale/graph.h @@ -42,15 +42,17 @@ typedef struct SwsGraph SwsGraph; * Output `h` lines of filtered data. `out` and `in` point to the * start of the image buffer for this pass. */ -typedef void (*sws_filter_run_t)(const AVFrame *out, const AVFrame *in, +typedef void (*sws_filter_run_t)(const SwsFrame *out, const SwsFrame *in, int y, int h, const SwsPass *pass); /** * Represents an allocated output buffer for a filter pass. */ typedef struct SwsPassBuffer { + SwsFrame frame; + int width, height; /* dimensions of this buffer */ - AVFrame *frame; + AVFrame *avframe; /* backing storage for `frame` */ } SwsPassBuffer; /** @@ -86,7 +88,7 @@ struct SwsPass { /** * Called once from the main thread before running the filter. Optional. */ - void (*setup)(const AVFrame *out, const AVFrame *in, const SwsPass *pass); + void (*setup)(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass); /** * Optional private state and associated free() function. @@ -123,20 +125,14 @@ typedef struct SwsGraph { SwsFormat src, dst; int field; - /** - * Temporary storage to hold individual fields of the input frames. - * No actual ownership over the data. - */ - AVFrame *field_tmp[2]; - /** * Temporary execution state inside ff_sws_graph_run(); used to pass * data to worker threads. */ struct { const SwsPass *pass; /* current filter pass */ - const AVFrame *input; /* current filter pass input/output */ - const AVFrame *output; + const SwsFrame *input; /* current filter pass input/output */ + const SwsFrame *output; } exec; } SwsGraph; diff --git a/libswscale/ops.c b/libswscale/ops.c index b3db758682..9f53657d8a 100644 --- a/libswscale/ops.c +++ b/libswscale/ops.c @@ -945,7 +945,7 @@ static inline void get_row_data(const SwsOpPass *p, const int y, out[i] = base->out[i] + (y >> base->out_sub_y[i]) * base->out_stride[i]; } -static void op_pass_setup(const AVFrame *out, const AVFrame *in, +static void op_pass_setup(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass) { const AVPixFmtDescriptor *indesc = av_pix_fmt_desc_get(in->format); @@ -1009,8 +1009,8 @@ static void op_pass_setup(const AVFrame *out, const AVFrame *in, exec->out_bump[i] = exec->out_stride[i] - blocks_main * exec->block_size_out; } - exec->src_frame_ptr = in; - exec->dst_frame_ptr = out; + exec->in_frame = in; + exec->out_frame = out; } /* Dispatch kernel over the last column of the image using memcpy */ @@ -1079,7 +1079,7 @@ handle_tail(const SwsOpPass *p, SwsOpExec *exec, } } -static void op_pass_run(const AVFrame *out, const AVFrame *in, const int y, +static void op_pass_run(const SwsFrame *out, const SwsFrame *in, const int y, const int h, const SwsPass *pass) { const SwsOpPass *p = pass->priv; diff --git a/libswscale/ops_internal.h b/libswscale/ops_internal.h index e07ef5285f..808b9ba4e0 100644 --- a/libswscale/ops_internal.h +++ b/libswscale/ops_internal.h @@ -80,8 +80,9 @@ typedef struct SwsOpExec { uint8_t in_sub_y[4], out_sub_y[4]; uint8_t in_sub_x[4], out_sub_x[4]; - const AVFrame *src_frame_ptr; - const AVFrame *dst_frame_ptr; + /* Pointers back to the original SwsFrame */ + const SwsFrame *in_frame; + const SwsFrame *out_frame; } SwsOpExec; static_assert(sizeof(SwsOpExec) == 24 * sizeof(void *) + diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index b77eb924c7..b0ff1dc6f3 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -100,8 +100,8 @@ static void process(const SwsOpExec *exec, const void *priv, FFVulkanFunctions *vk = &p->s->vkctx.vkfn; ff_vk_exec_start(&p->s->vkctx, ec); - AVFrame *src_f = (AVFrame *)exec->src_frame_ptr; - AVFrame *dst_f = (AVFrame *)exec->dst_frame_ptr; + AVFrame *src_f = (AVFrame *) exec->in_frame->avframe; + AVFrame *dst_f = (AVFrame *) exec->out_frame->avframe; ff_vk_exec_add_dep_frame(&p->s->vkctx, ec, src_f, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
