Module: Mesa Branch: main Commit: 32fbd388895e4590488a63417c36dc342d1737a7 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=32fbd388895e4590488a63417c36dc342d1737a7
Author: Louis-Francis Ratté-Boulianne <[email protected]> Date: Mon Sep 11 02:52:16 2023 -0400 panfrost: Add method to get size of AFBC superblocks valid data The compute shader is going through all the AFBC header blocks and calculating the size of valid data by summing up the subblock sizes (and taking into account solid color, uncompressed mode and copy blocks). The result is written into the given buffer (array of `pan_afbc_block_info`). The size is rounded up to the alignment parameter directly in the shader. Signed-off-by: Louis-Francis Ratté-Boulianne <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25012> --- src/gallium/drivers/panfrost/pan_afbc_cso.c | 95 ++++++++++++++++++++++++++++ src/gallium/drivers/panfrost/pan_afbc_cso.h | 11 ++++ src/gallium/drivers/panfrost/pan_cmdstream.c | 21 ++++++ src/gallium/drivers/panfrost/pan_resource.c | 34 ++++++++++ src/gallium/drivers/panfrost/pan_resource.h | 4 ++ src/gallium/drivers/panfrost/pan_screen.h | 6 ++ 6 files changed, 171 insertions(+) diff --git a/src/gallium/drivers/panfrost/pan_afbc_cso.c b/src/gallium/drivers/panfrost/pan_afbc_cso.c index a57d1021803..62597b31dcc 100644 --- a/src/gallium/drivers/panfrost/pan_afbc_cso.c +++ b/src/gallium/drivers/panfrost/pan_afbc_cso.c @@ -42,6 +42,99 @@ nir_imm_int(b, offsetof(struct panfrost_afbc_##name##_info, field)), \ .align_mul = 4, .range = ~0) +static nir_def * +read_afbc_header(nir_builder *b, nir_def *buf, nir_def *idx) +{ + nir_def *offset = nir_imul_imm(b, idx, AFBC_HEADER_BYTES_PER_TILE); + return nir_load_global(b, nir_iadd(b, buf, nir_u2u64(b, offset)), 16, + AFBC_HEADER_BYTES_PER_TILE / 4, 32); +} + +static nir_def * +get_superblock_size(nir_builder *b, unsigned arch, nir_def *hdr, + nir_def *uncompressed_size) +{ + nir_def *size = nir_imm_int(b, 0); + + unsigned body_base_ptr_len = 32; + unsigned nr_subblocks = 16; + unsigned sz_len = 6; /* bits */ + nir_def *words[4]; + nir_def *mask = nir_imm_int(b, (1 << sz_len) - 1); + nir_def *is_solid_color = nir_imm_bool(b, false); + + for (int i = 0; i < 4; i++) + words[i] = nir_channel(b, hdr, i); + + /* Sum up all of the subblock sizes */ + for (int i = 0; i < nr_subblocks; i++) { + nir_def *subblock_size; + unsigned bitoffset = body_base_ptr_len + (i * sz_len); + unsigned start = bitoffset / 32; + unsigned end = (bitoffset + (sz_len - 1)) / 32; + unsigned offset = bitoffset % 32; + + /* Handle differently if the size field is split between two words + * of the header */ + if (start != end) { + subblock_size = nir_ior(b, nir_ushr_imm(b, words[start], offset), + nir_ishl_imm(b, words[end], 32 - offset)); + subblock_size = nir_iand(b, subblock_size, mask); + } else { + subblock_size = + nir_ubitfield_extract_imm(b, words[start], offset, sz_len); + } + subblock_size = nir_bcsel(b, nir_ieq_imm(b, subblock_size, 1), + uncompressed_size, subblock_size); + size = nir_iadd(b, size, subblock_size); + + /* When the first subblock size is set to zero, the whole superblock is + * filled with a solid color specified in the header */ + if (arch >= 7 && i == 0) + is_solid_color = nir_ieq_imm(b, size, 0); + } + + return (arch >= 7) + ? nir_bcsel(b, is_solid_color, nir_imm_zero(b, 1, 32), size) + : size; +} + +#define panfrost_afbc_size_get_info_field(b, field) \ + panfrost_afbc_get_info_field(size, b, field) + +static nir_shader * +panfrost_afbc_create_size_shader(struct panfrost_screen *screen, unsigned bpp, + unsigned align) +{ + struct panfrost_device *dev = pan_device(&screen->base); + + nir_builder b = nir_builder_init_simple_shader( + MESA_SHADER_COMPUTE, screen->vtbl.get_compiler_options(), + "panfrost_afbc_size(bpp=%d)", bpp); + + panfrost_afbc_add_info_ubo(size, b); + + nir_def *coord = nir_load_global_invocation_id(&b, 32); + nir_def *block_idx = nir_channel(&b, coord, 0); + nir_def *src = panfrost_afbc_size_get_info_field(&b, src); + nir_def *metadata = panfrost_afbc_size_get_info_field(&b, metadata); + nir_def *uncompressed_size = nir_imm_int(&b, 4 * 4 * bpp / 8); /* bytes */ + + nir_def *hdr = read_afbc_header(&b, src, block_idx); + nir_def *size = get_superblock_size(&b, dev->arch, hdr, uncompressed_size); + size = nir_iand(&b, nir_iadd(&b, size, nir_imm_int(&b, align - 1)), + nir_inot(&b, nir_imm_int(&b, align - 1))); + + nir_def *offset = nir_u2u64( + &b, + nir_iadd(&b, + nir_imul_imm(&b, block_idx, sizeof(struct pan_afbc_block_info)), + nir_imm_int(&b, offsetof(struct pan_afbc_block_info, size)))); + nir_store_global(&b, nir_iadd(&b, metadata, offset), 4, size, 0x1); + + return b.shader; +} + struct pan_afbc_shader_data * panfrost_afbc_get_shaders(struct panfrost_context *ctx, struct panfrost_resource *rsrc, unsigned align) @@ -77,6 +170,8 @@ panfrost_afbc_get_shaders(struct panfrost_context *ctx, shader->name##_cso = pctx->create_compute_state(pctx, &cso); \ } + COMPILE_SHADER(size, key.bpp, key.align); + #undef COMPILE_SHADER pthread_mutex_lock(&ctx->afbc_shaders.lock); diff --git a/src/gallium/drivers/panfrost/pan_afbc_cso.h b/src/gallium/drivers/panfrost/pan_afbc_cso.h index a9616829364..86e09c57735 100644 --- a/src/gallium/drivers/panfrost/pan_afbc_cso.h +++ b/src/gallium/drivers/panfrost/pan_afbc_cso.h @@ -41,6 +41,7 @@ struct pan_afbc_shader_key { struct pan_afbc_shader_data { struct pan_afbc_shader_key key; + void *size_cso; }; struct pan_afbc_shaders { @@ -48,6 +49,16 @@ struct pan_afbc_shaders { pthread_mutex_t lock; }; +struct pan_afbc_block_info { + uint32_t size; + uint32_t offset; +}; + +struct panfrost_afbc_size_info { + mali_ptr src; + mali_ptr metadata; +} PACKED; + void panfrost_afbc_context_init(struct panfrost_context *ctx); void panfrost_afbc_context_destroy(struct panfrost_context *ctx); diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.c b/src/gallium/drivers/panfrost/pan_cmdstream.c index 8052e1289b3..c2b1a69d4ca 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.c +++ b/src/gallium/drivers/panfrost/pan_cmdstream.c @@ -1,4 +1,5 @@ /* + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. * Copyright (C) 2018 Alyssa Rosenzweig * Copyright (C) 2020 Collabora Ltd. * Copyright © 2017 Intel Corporation @@ -37,6 +38,7 @@ #include "genxml/gen_macros.h" +#include "pan_afbc_cso.h" #include "pan_blend.h" #include "pan_blitter.h" #include "pan_bo.h" @@ -3933,6 +3935,24 @@ panfrost_launch_afbc_shader(struct panfrost_batch *batch, void *cso, panfrost_launch_afbc_shader(batch, shaders->name##_cso, &constant_buffer, \ nr_blocks); +static void +panfrost_afbc_size(struct panfrost_batch *batch, struct panfrost_resource *src, + struct panfrost_bo *metadata, unsigned offset, + unsigned level) +{ + struct pan_image_slice_layout *slice = &src->image.layout.slices[level]; + struct panfrost_afbc_size_info consts = { + .src = + src->image.data.bo->ptr.gpu + src->image.data.offset + slice->offset, + .metadata = metadata->ptr.gpu + offset, + }; + + panfrost_batch_read_rsrc(batch, src, PIPE_SHADER_COMPUTE); + panfrost_batch_write_bo(batch, metadata, PIPE_SHADER_COMPUTE); + + LAUNCH_AFBC_SHADER(size, batch, src, consts, slice->afbc.nr_blocks); +} + static void * panfrost_create_rasterizer_state(struct pipe_context *pctx, const struct pipe_rasterizer_state *cso) @@ -4549,6 +4569,7 @@ GENX(panfrost_cmdstream_screen_init)(struct panfrost_screen *screen) screen->vtbl.init_polygon_list = init_polygon_list; screen->vtbl.get_compiler_options = GENX(pan_shader_get_compiler_options); screen->vtbl.compile_shader = GENX(pan_shader_compile); + screen->vtbl.afbc_size = panfrost_afbc_size; GENX(pan_blitter_init) (dev, &screen->blitter.bin_pool.base, &screen->blitter.desc_pool.base); diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c index fd330a793dc..b628670835f 100644 --- a/src/gallium/drivers/panfrost/pan_resource.c +++ b/src/gallium/drivers/panfrost/pan_resource.c @@ -4,6 +4,7 @@ * Copyright (C) 2014-2017 Broadcom * Copyright (C) 2018-2019 Alyssa Rosenzweig * Copyright (C) 2019 Collabora, Ltd. + * Copyright (C) 2023 Amazon.com, Inc. or its affiliates * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -1269,6 +1270,39 @@ panfrost_should_linear_convert(struct panfrost_device *dev, } } +struct panfrost_bo * +panfrost_get_afbc_superblock_sizes(struct panfrost_context *ctx, + struct panfrost_resource *rsrc, + unsigned first_level, unsigned last_level, + unsigned *out_offsets) +{ + struct panfrost_screen *screen = pan_screen(ctx->base.screen); + struct panfrost_device *dev = pan_device(ctx->base.screen); + struct panfrost_batch *batch; + struct panfrost_bo *bo; + unsigned metadata_size = 0; + + for (int level = first_level; level <= last_level; ++level) { + struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level]; + unsigned sz = slice->afbc.nr_blocks * sizeof(struct pan_afbc_block_info); + out_offsets[level - first_level] = metadata_size; + metadata_size += sz; + } + + panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC before size flush"); + batch = panfrost_get_fresh_batch_for_fbo(ctx, "AFBC superblock sizes"); + bo = panfrost_bo_create(dev, metadata_size, 0, "AFBC superblock sizes"); + + for (int level = first_level; level <= last_level; ++level) { + unsigned offset = out_offsets[level - first_level]; + screen->vtbl.afbc_size(batch, rsrc, bo, offset, level); + } + + panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC after size flush"); + + return bo; +} + static void panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer) { diff --git a/src/gallium/drivers/panfrost/pan_resource.h b/src/gallium/drivers/panfrost/pan_resource.h index 37bb3ccddc7..e2bdd202823 100644 --- a/src/gallium/drivers/panfrost/pan_resource.h +++ b/src/gallium/drivers/panfrost/pan_resource.h @@ -177,6 +177,10 @@ panfrost_resource_create_with_modifier(struct pipe_screen *screen, const struct pipe_resource *template, uint64_t modifier); +struct panfrost_bo *panfrost_get_afbc_superblock_sizes( + struct panfrost_context *ctx, struct panfrost_resource *rsrc, + unsigned first_level, unsigned last_level, unsigned *out_offsets); + void pan_resource_modifier_convert(struct panfrost_context *ctx, struct panfrost_resource *rsrc, uint64_t modifier, const char *reason); diff --git a/src/gallium/drivers/panfrost/pan_screen.h b/src/gallium/drivers/panfrost/pan_screen.h index 8c1f7412c8a..3f5c690d4b0 100644 --- a/src/gallium/drivers/panfrost/pan_screen.h +++ b/src/gallium/drivers/panfrost/pan_screen.h @@ -99,6 +99,12 @@ struct panfrost_vtable { void (*compile_shader)(nir_shader *s, struct panfrost_compile_inputs *inputs, struct util_dynarray *binary, struct pan_shader_info *info); + + /* Run a compute shader to get the compressed size of each superblock */ + void (*afbc_size)(struct panfrost_batch *batch, + struct panfrost_resource *src, + struct panfrost_bo *metadata, unsigned offset, + unsigned level); }; struct panfrost_screen {
