When enabled, dynfield3 is reserved for dynamic fields registered with
RTE_MBUF_DYNFIELD_F_NO_COPY. Such fields are allocated from dynfield3
and are not copied by the generic mbuf dynamic-field copy helper used by
mbuf copy and clone operations. Dynamic fields registered without this
flag continue to use the existing copied dynamic-field storage and are
prevented from overlapping dynfield3.
Disable the octeontx mempool driver when the option is enabled because
it requires sizeof(struct rte_mbuf) to remain at most 128 bytes.
Signed-off-by: Randy L Tice <[email protected]>
---
app/test/test_mbuf.c | 72 ++++++++++++++++++++++++--
config/meson.build | 5 ++
devtools/test-meson-builds.sh | 3 ++
doc/guides/prog_guide/mbuf_lib.rst | 9 ++++
doc/guides/rel_notes/release_26_11.rst | 17 +++++-
drivers/mempool/octeontx/meson.build | 5 ++
lib/mbuf/rte_mbuf_core.h | 6 +++
lib/mbuf/rte_mbuf_dyn.c | 54 ++++++++++++++++---
lib/mbuf/rte_mbuf_dyn.h | 11 +++-
meson_options.txt | 2 +
10 files changed, 172 insertions(+), 12 deletions(-)
diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c
index db23259745..c03878bd96 100644
--- a/app/test/test_mbuf.c
+++ b/app/test/test_mbuf.c
@@ -2569,7 +2569,7 @@ test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
};
const struct rte_mbuf_dynfield dynfield_fail_big = {
.name = "test-dynfield-fail-big",
- .size = 256,
+ .size = sizeof(struct rte_mbuf),
.align = 1,
.flags = 0,
};
@@ -2583,8 +2583,28 @@ test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
.name = "test-dynfield",
.size = sizeof(uint8_t),
.align = alignof(uint8_t),
- .flags = 1,
+ .flags = RTE_MBUF_DYNFIELD_F_NO_COPY << 1,
+ };
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ const struct rte_mbuf_dynfield dynfield3_no_copy = {
+ .name = "test-dynfield3-no-copy",
+ .size = sizeof(uint64_t),
+ .align = alignof(uint64_t),
+ .flags = RTE_MBUF_DYNFIELD_F_NO_COPY,
+ };
+ const struct rte_mbuf_dynfield dynfield_no_copy_bad_offset = {
+ .name = "test-dynfield-no-copy-bad-offset",
+ .size = sizeof(uint64_t),
+ .align = alignof(uint64_t),
+ .flags = RTE_MBUF_DYNFIELD_F_NO_COPY,
+ };
+ const struct rte_mbuf_dynfield dynfield_copy_bad_offset = {
+ .name = "test-dynfield-copy-bad-offset",
+ .size = 2 * sizeof(uint64_t),
+ .align = alignof(uint64_t),
+ .flags = 0,
};
+#endif
const struct rte_mbuf_dynflag dynflag_fail_flag = {
.name = "test-dynflag",
.flags = 1,
@@ -2602,7 +2622,11 @@ test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
.flags = 0,
};
struct rte_mbuf *m = NULL;
+ struct rte_mbuf *mc = NULL;
int offset, offset2, offset3;
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ int dynfield3_no_copy_offset;
+#endif
int flag, flag2, flag3;
int ret;
@@ -2654,6 +2678,29 @@ test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
if (ret != -1)
GOTO_FAIL("dynamic field creation should fail (invalid flag)");
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ dynfield3_no_copy_offset =
rte_mbuf_dynfield_register_offset(&dynfield3_no_copy,
+ offsetof(struct rte_mbuf, dynfield3));
+ if (dynfield3_no_copy_offset != offsetof(struct rte_mbuf, dynfield3))
+ GOTO_FAIL("failed to register no-copy dynfield3 field, offset=%d:
%s",
+ dynfield3_no_copy_offset, strerror(errno));
+
+ ret = rte_mbuf_dynfield_register_offset(&dynfield_no_copy_bad_offset,
+ offsetof(struct rte_mbuf, dynfield1[0]));
+ if (ret != -1)
+ GOTO_FAIL("no-copy dynamic field creation should fail outside
dynfield3");
+
+ ret = rte_mbuf_dynfield_register_offset(&dynfield_copy_bad_offset,
+ offsetof(struct rte_mbuf, dynfield3));
+ if (ret != -1)
+ GOTO_FAIL("copied dynamic field creation should fail in
dynfield3");
+
+ ret = rte_mbuf_dynfield_register_offset(&dynfield_copy_bad_offset,
+ offsetof(struct rte_mbuf, dynfield3) -
sizeof(uint64_t));
+ if (ret != -1)
+ GOTO_FAIL("copied dynamic field creation should fail when straddling
dynfield3");
+#endif
+
ret = rte_mbuf_dynflag_register(&dynflag_fail_flag);
if (ret != -1)
GOTO_FAIL("dynamic flag creation should fail (invalid flag)");
@@ -2693,13 +2740,29 @@ test_mbuf_dyn(struct rte_mempool *pktmbuf_pool)
if (*RTE_MBUF_DYNFIELD(m, offset2, uint16_t *) != 1000)
GOTO_FAIL("failed to read dynamic field");
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ *RTE_MBUF_DYNFIELD(m, dynfield3_no_copy_offset, uint64_t *) =
+ UINT64_C(0x8877665544332211);
+ mc = rte_pktmbuf_alloc(pktmbuf_pool);
+ if (mc == NULL)
+ GOTO_FAIL("Cannot allocate mbuf for dynamic field copy test");
+ *RTE_MBUF_DYNFIELD(mc, dynfield3_no_copy_offset, uint64_t *) =
+ UINT64_C(0xa5a5a5a5a5a5a5a5);
+ rte_mbuf_dynfield_copy(mc, m);
+ if (*RTE_MBUF_DYNFIELD(mc, dynfield3_no_copy_offset, uint64_t *) !=
+ UINT64_C(0xa5a5a5a5a5a5a5a5))
+ GOTO_FAIL("copied no-copy dynfield3 dynamic field");
+#endif
+
/* set a dynamic flag */
m->ol_flags |= (1ULL << flag);
rte_mbuf_dyn_dump(stdout);
+ rte_pktmbuf_free(mc);
rte_pktmbuf_free(m);
return 0;
fail:
+ rte_pktmbuf_free(mc);
rte_pktmbuf_free(m);
return -1;
}
@@ -2776,8 +2839,9 @@ test_mbuf(void)
struct rte_mempool *pktmbuf_pool = NULL;
struct rte_mempool *pktmbuf_pool2 = NULL;
-
- RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) != RTE_CACHE_LINE_MIN_SIZE *
2);
+ RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) !=
+ RTE_CACHE_LINE_MIN_SIZE * 2 +
+ RTE_MBUF_DYNFIELD3_SIZE);
/* create pktmbuf pool if it does not exist */
pktmbuf_pool = rte_pktmbuf_pool_create("test_pktmbuf_pool",
diff --git a/config/meson.build b/config/meson.build
index 344f68822b..a6bd25b9ac 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -384,6 +384,8 @@ dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
+mbuf_dynfield3_size = get_option('mbuf_dynfield3_size')
+dpdk_conf.set('RTE_MBUF_DYNFIELD3_SIZE', mbuf_dynfield3_size)
# values which have defaults which may be overridden
dpdk_conf.set('RTE_MAX_VFIO_GROUPS', 64)
dpdk_conf.set('RTE_DRIVER_MEMPOOL_BUCKET_SIZE_KB', 64)
@@ -395,6 +397,9 @@ dpdk_conf.set10('RTE_IOVA_IN_MBUF',
get_option('enable_iova_as_pa'))
compile_time_cpuflags = []
subdir(arch_subdir)
+if mbuf_dynfield3_size % dpdk_conf.get('RTE_CACHE_LINE_SIZE') != 0
+ error('mbuf_dynfield3_size must be a multiple of RTE_CACHE_LINE_SIZE')
+endif
dpdk_conf.set('RTE_COMPILE_TIME_CPUFLAGS', ','.join(compile_time_cpuflags))
# apply cross-specific options
diff --git a/devtools/test-meson-builds.sh b/devtools/test-meson-builds.sh
index 11e4be3f88..23115def8c 100755
--- a/devtools/test-meson-builds.sh
+++ b/devtools/test-meson-builds.sh
@@ -259,6 +259,9 @@ fi
build build-x86-generic cc skipABI --buildtype=debug -Dcheck_includes=true \
-Dlibdir=lib -Dcpu_instruction_set=$generic_isa $use_shared
+build build-mbuf-dynfield3 cc skipABI --buildtype=debug \
+ -Dmbuf_dynfield3_size=256 $use_shared
+
# 32-bit with default compiler
if check_cc_flags '-m32' ; then
target_override='i386-pc-linux-gnu'
diff --git a/doc/guides/prog_guide/mbuf_lib.rst
b/doc/guides/prog_guide/mbuf_lib.rst
index cf64add109..ca0efb99c1 100644
--- a/doc/guides/prog_guide/mbuf_lib.rst
+++ b/doc/guides/prog_guide/mbuf_lib.rst
@@ -234,6 +234,15 @@ The dynamic fields and flags are managed with the
functions ``rte_mbuf_dyn*``.
It is not possible to unregister fields or flags.
+The build option ``mbuf_dynfield3_size`` can add extra cache-line-aligned
+dynamic field storage to ``struct rte_mbuf``. This increases every mbuf by
+the configured amount and changes the mbuf layout, so applications and
+secondary processes must be built with the same value as the primary process.
+The option defaults to ``0``. The extra storage is reserved for dynamic
+fields registered with ``RTE_MBUF_DYNFIELD_F_NO_COPY``. These fields are not
+copied by mbuf copy and clone operations. Dynamic fields registered without
+this flag continue to use the existing copied dynamic-field storage.
+
.. _direct_indirect_buffer:
Direct and Indirect Buffers
diff --git a/doc/guides/rel_notes/release_26_11.rst
b/doc/guides/rel_notes/release_26_11.rst
index dec96ccbc7..cff68d7034 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -60,6 +60,22 @@ New Features
Added the experimental ``rte_cpu_socket_id()`` function
to map an OS logical CPU ID to the NUMA socket containing that CPU.
+* **Added optional extra mbuf dynamic field storage.**
+
+ Added ``mbuf_dynfield3_size`` build option to enable a
+ cache-line-aligned ``dynfield3`` area in ``struct rte_mbuf``.
+ The configured size is defined as ``RTE_MBUF_DYNFIELD3_SIZE``
+ in ``rte_build_config.h``.
+ The extra storage is reserved for dynamic fields registered with
+ ``RTE_MBUF_DYNFIELD_F_NO_COPY``.
+ These fields are not copied by generic mbuf copy and clone operations.
+ Dynamic fields registered without this flag continue to use the existing
+ copied dynamic-field storage.
+ Applications and secondary processes must be built with the same
+ non-zero value as the primary process.
+ The ``mempool/octeontx`` driver is disabled when this option is
+ enabled because it requires a fixed 128-byte mbuf header.
+
* **Added TPID support to VLAN tag insertion.**
Added ``rte_vlan_insert_tpid()`` to the net library.
@@ -338,7 +354,6 @@ Known Issues
Also, make sure to start the actual text at the margin.
=======================================================
-
Tested Platforms
----------------
diff --git a/drivers/mempool/octeontx/meson.build b/drivers/mempool/octeontx/meson.build
index 3ccecac75d..01aece94be 100644
--- a/drivers/mempool/octeontx/meson.build
+++ b/drivers/mempool/octeontx/meson.build
@@ -6,6 +6,11 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
reason = 'only supported on 64-bit Linux'
subdir_done()
endif
+if get_option('mbuf_dynfield3_size') != 0
+ build = false
+ reason = 'requires sizeof(struct rte_mbuf) <= 128'
+ subdir_done()
+endif
sources = files(
'octeontx_fpavf.c',
diff --git a/lib/mbuf/rte_mbuf_core.h b/lib/mbuf/rte_mbuf_core.h
index 98b0bd9ca7..b4f204b268 100644
--- a/lib/mbuf/rte_mbuf_core.h
+++ b/lib/mbuf/rte_mbuf_core.h
@@ -686,6 +686,12 @@ struct __rte_cache_aligned rte_mbuf {
uint16_t timesync;
uint32_t dynfield1[9]; /**< Reserved for dynamic fields. */
+
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ alignas(RTE_CACHE_LINE_SIZE)
+ uint64_t dynfield3[RTE_MBUF_DYNFIELD3_SIZE / sizeof(uint64_t)];
+ /**< Reserved cache-line-aligned space for dynamic fields. */
+#endif /* RTE_MBUF_DYNFIELD3_SIZE > 0 */
};
/**
diff --git a/lib/mbuf/rte_mbuf_dyn.c b/lib/mbuf/rte_mbuf_dyn.c
index 5987c9dee8..7cecbc9ee6 100644
--- a/lib/mbuf/rte_mbuf_dyn.c
+++ b/lib/mbuf/rte_mbuf_dyn.c
@@ -3,6 +3,7 @@
*/
#include <stdalign.h>
+#include <stddef.h>
#include <sys/queue.h>
#include <stdint.h>
#include <limits.h>
@@ -51,7 +52,7 @@ struct mbuf_dyn_shm {
* The value is the size of the biggest aligned element that
* can fit in the zone.
*/
- uint8_t free_space[sizeof(struct rte_mbuf)];
+ uint16_t free_space[sizeof(struct rte_mbuf)];
/** Bitfield of available flags. */
uint64_t free_flags;
};
@@ -135,6 +136,9 @@ init_shared_mem(void)
#if !RTE_IOVA_IN_MBUF
mark_free(dynfield2);
#endif
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ mark_free(dynfield3);
+#endif
/* init free_flags */
for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask
<<= 1)
@@ -147,11 +151,48 @@ init_shared_mem(void)
}
/* check if this offset can be used */
+static bool
+dynfield_in_dynfield3(size_t offset, size_t size)
+{
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ size_t dynfield3_offset = offsetof(struct rte_mbuf, dynfield3);
+
+ return offset >= dynfield3_offset &&
+ size <= sizeof(((struct rte_mbuf *)0)->dynfield3) &&
+ offset - dynfield3_offset <= sizeof(((struct rte_mbuf
*)0)->dynfield3) - size;
+#else
+ RTE_SET_USED(offset);
+ RTE_SET_USED(size);
+ return false;
+#endif
+}
+
+static bool
+dynfield_overlaps_dynfield3(size_t offset, size_t size)
+{
+#if RTE_MBUF_DYNFIELD3_SIZE > 0
+ size_t dynfield3_offset = offsetof(struct rte_mbuf, dynfield3);
+ size_t dynfield3_end = dynfield3_offset + sizeof(((struct rte_mbuf
*)0)->dynfield3);
+
+ return offset < dynfield3_end && offset + size > dynfield3_offset;
+#else
+ RTE_SET_USED(offset);
+ RTE_SET_USED(size);
+ return false;
+#endif
+}
+
static int
-check_offset(size_t offset, size_t size, size_t align)
+check_offset(size_t offset, size_t size, size_t align, unsigned int flags)
{
size_t i;
+ if ((flags & RTE_MBUF_DYNFIELD_F_NO_COPY) != 0 &&
+ !dynfield_in_dynfield3(offset, size))
+ return -1;
+ if ((flags & RTE_MBUF_DYNFIELD_F_NO_COPY) == 0 &&
+ dynfield_overlaps_dynfield3(offset, size))
+ return -1;
if ((offset & (align - 1)) != 0)
return -1;
if (offset + size > sizeof(struct rte_mbuf))
@@ -268,7 +309,7 @@ __rte_mbuf_dynfield_register_offset(const struct
rte_mbuf_dynfield *params,
offset < sizeof(struct rte_mbuf);
offset++) {
if (check_offset(offset, params->size,
- params->align) == 0 &&
+ params->align, params->flags) == 0
&&
shm->free_space[offset] < best_zone) {
best_zone = shm->free_space[offset];
req = offset;
@@ -279,7 +320,8 @@ __rte_mbuf_dynfield_register_offset(const struct
rte_mbuf_dynfield *params,
return -1;
}
} else {
- if (check_offset(req, params->size, params->align) < 0) {
+ if (check_offset(req, params->size, params->align,
+ params->flags) < 0) {
rte_errno = EBUSY;
return -1;
}
@@ -342,7 +384,7 @@ rte_mbuf_dynfield_register_offset(const struct
rte_mbuf_dynfield *params,
rte_errno = EINVAL;
return -1;
}
- if (params->flags != 0) {
+ if ((params->flags & ~RTE_MBUF_DYNFIELD_F_NO_COPY) != 0) {
rte_errno = EINVAL;
return -1;
}
@@ -573,7 +615,7 @@ void rte_mbuf_dyn_dump(FILE *out)
for (i = 0; i < sizeof(struct rte_mbuf); i++) {
if ((i % 8) == 0)
fprintf(out, " %4.4zx: ", i);
- fprintf(out, "%2.2x%s", shm->free_space[i],
+ fprintf(out, "%4.4x%s", shm->free_space[i],
(i % 8 != 7) ? " " : "\n");
}
fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n");
diff --git a/lib/mbuf/rte_mbuf_dyn.h b/lib/mbuf/rte_mbuf_dyn.h
index 20ce505bb4..d9a46b3c33 100644
--- a/lib/mbuf/rte_mbuf_dyn.h
+++ b/lib/mbuf/rte_mbuf_dyn.h
@@ -69,6 +69,7 @@
#include <stdio.h>
#include <stdint.h>
+#include <rte_bitops.h>
#include <rte_stdatomic.h>
#ifdef __cplusplus
@@ -80,6 +81,14 @@ extern "C" {
*/
#define RTE_MBUF_DYN_NAMESIZE 64
+/**
+ * Do not copy this dynamic field during mbuf clone or copy.
+ *
+ * Fields using this flag are allocated from the optional dynfield3 area
+ * configured by the mbuf_dynfield3_size build option.
+ */
+#define RTE_MBUF_DYNFIELD_F_NO_COPY RTE_BIT32(0)
+
/**
* Structure describing the parameters of a mbuf dynamic field.
*/
@@ -87,7 +96,7 @@ struct rte_mbuf_dynfield {
char name[RTE_MBUF_DYN_NAMESIZE]; /**< Name of the field. */
size_t size; /**< The number of bytes to reserve. */
size_t align; /**< The alignment constraint (power of 2). */
- unsigned int flags; /**< Reserved for future use, must be 0. */
+ unsigned int flags; /**< Dynamic field flags. */
};
/**
diff --git a/meson_options.txt b/meson_options.txt
index e28d24054c..1248498a3d 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -44,6 +44,8 @@ option('max_numa_nodes', type: 'string', value: 'default',
description:
'Set the highest NUMA node supported by EAL; "default" is different per-arch,
"detect" detects the highest NUMA node on the build machine.')
option('enable_iova_as_pa', type: 'boolean', value: true, description:
'Support the use of physical addresses for IO addresses, such as used
by UIO or VFIO in no-IOMMU mode. When disabled, DPDK can only run with IOMMU
support for address mappings, but will have more space available in the mbuf
structure.')
+option('mbuf_dynfield3_size', type: 'integer', min: 0, value: 0, description:
+ 'Size of optional extra mbuf dynamic field area, in bytes.')
option('mbuf_refcnt_atomic', type: 'boolean', value: true, description:
'Atomically access the mbuf refcnt.')
option('platform', type: 'string', value: 'native', description:
