On Tue, Jul 28, 2026 at 7:58 AM Leon Hwang <[email protected]> wrote: > > Use aligned nofault word loads for the strchr, strrchr, and strlen > kfunc families. Align once, hoist the full-word boundary, and use > word-at-a-time masks to skip irrelevant words or locate the first > matching or NUL byte. > > Factor the common unaligned-byte, aligned-word, and byte-fallback > plumbing into an expansion macro. Keep algorithm-specific actions expanded > at each call site. > > Allow the shared finder to match both ASCII case variants so the > case-insensitive substring kfuncs can use the same word-at-a-time scan. > > If a word load faults, retry from the same address byte-at-a-time. Keep > the byte path under KMSAN to avoid reading uninitialized storage beyond > the terminator, and preserve the existing limits and error semantics. > > Assisted-by: Codex:gpt-5.6-sol > Signed-off-by: Leon Hwang <[email protected]> > --- > kernel/bpf/helpers.c | 180 +++++++++++++++++++++++++++++++++---------- > 1 file changed, 141 insertions(+), 39 deletions(-) > > diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c > index 88b38db47de9..36075c683166 100644 > --- a/kernel/bpf/helpers.c > +++ b/kernel/bpf/helpers.c > @@ -30,6 +30,8 @@ > #include <linux/irq_work.h> > #include <linux/buildid.h> > > +#include <asm/word-at-a-time.h> > + > #include "../../lib/kstrtox.h" > > /* If kernel subsystem is allowing eBPF programs to call this function, > @@ -3727,6 +3729,95 @@ __bpf_kfunc void __bpf_trap(void) > * __get_kernel_nofault instead of plain dereference to make them safe. > */ > > +/* Abstract the common unaligned-byte, aligned-word, and byte-fallback scan. > */ > +#define bpf_str_for_each_word(s, limit, pos, word, byte, byte_label, \ > + byte_action, word_action, err_label) \ > +do { \ > + __label__ byte_label; \ > + size_t __word_end; \ > + \ > + if (IS_ENABLED(CONFIG_KMSAN)) \ > + goto byte_label; \ > + \ > + for (; (pos) < (limit) && \ > + !IS_ALIGNED((unsigned long)((s) + (pos)), sizeof(word)); \ > + (pos)++) { \ > + __get_kernel_nofault(&(byte), (s) + (pos), \ > + unsigned char, err_label); \ > + byte_action; \ > + } \ > + \ > + __word_end = (pos) + round_down((limit) - (pos), sizeof(word)); \ > + for (; (pos) < __word_end; (pos) += sizeof(word)) { \ > + __get_kernel_nofault(&(word), (s) + (pos), \ > + unsigned long, byte_label); \ > + word_action; \ > + } \ > + \ > +byte_label: \ > + for (; (pos) < (limit); (pos)++) { \ > + __get_kernel_nofault(&(byte), (s) + (pos), \ > + unsigned char, err_label); \ > + byte_action; \ > + } \ > +} while (0) > + > +static __always_inline int bpf_str_find(const char *s, size_t limit, > unsigned char c, > + bool ignore_case, bool nul_is_match) > +{ > + const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; > + const unsigned char match_c = ignore_case ? tolower(c) : c; > + const unsigned char alt_c = ignore_case && islower(match_c) ? > + toupper(match_c) : match_c; > + const unsigned long repeated_c = REPEAT_BYTE(match_c); > + const unsigned long repeated_alt = REPEAT_BYTE(alt_c); > + unsigned long word, zero_data, char_data, alt_data; > + unsigned long zero_at, char_at, alt_at; > + const bool has_alt = alt_c != match_c; > + unsigned char sc; > + size_t pos = 0; > + > + bpf_str_for_each_word(s, limit, pos, word, sc, byte_at_a_time, ({ > + if (sc == match_c || (has_alt && sc == alt_c)) > + return pos; > + if (sc == '\0') > + return nul_is_match ? pos : -ENOENT; > + }), ({ > + zero_at = has_zero(word, &zero_data, &constants); > + char_at = has_zero(word ^ repeated_c, &char_data, &constants); > + alt_at = has_alt ? has_zero(word ^ repeated_alt, &alt_data, > &constants) : 0; > + if (!zero_at && !char_at && !alt_at) > + continue; > + > + if (zero_at) { > + zero_data = prep_zero_mask(word, zero_data, > &constants); > + zero_at = find_zero(create_zero_mask(zero_data)); > + } else { > + zero_at = sizeof(word); > + } > + if (char_at) { > + char_data = prep_zero_mask(word ^ repeated_c, > char_data, &constants); > + char_at = find_zero(create_zero_mask(char_data)); > + } else { > + char_at = sizeof(word); > + } > + if (alt_at) { > + alt_data = prep_zero_mask(word ^ repeated_alt, > alt_data, &constants); > + alt_at = find_zero(create_zero_mask(alt_data)); > + char_at = min(char_at, alt_at); > + } > + > + if (char_at <= zero_at) > + return pos + char_at; > + return nul_is_match ? pos + zero_at : -ENOENT; > + }), err_out);
looking at the above code, I'd say it's just not worth it. Too much code and complexity, IMO. For the absolute majority of BPF programs this small speed up won't matter, while for those BPF programs where doing tons of bpf_strstr-like operations is the essence of those programs and has a huge impact on the performance, they can basically implement and maintain this complexity in their own code base. > + > + return pos == XATTR_SIZE_MAX ? -E2BIG : -ENOENT; > + > +err_out: > + return -EFAULT; > +} > + [...]
