[I got another little block of time again and have a little more information.]
On Nov 18, 2025, at 10:39, Mark Millard <[email protected]> wrote: > On Nov 18, 2025, at 10:18, Mark Millard <[email protected]> wrote: > >> I modified system clang to not register its 2 SIGABRT handlers. >> >> The backtrace has the jemalloc call stack activity as well. >> >> (gdb) bt >> #0 thr_kill () at thr_kill.S:4 >> #1 0x2a08ef24 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:48 >> #2 0x2a145f38 in abort () at /usr/src/lib/libc/stdlib/abort.c:61 >> #3 0x2a196128 in ehooks_debug_zero_check (addr=addr@entry=0x2b629000, >> size=size@entry=12288) at >> /usr/src/contrib/jemalloc/include/jemalloc/internal/ehooks.h:170 >> #4 0x2a191f60 in ehooks_alloc (tsdn=0x2a2e4060, ehooks=0x2a600080, >> new_addr=0x0, size=<optimized out>, alignment=4096, zero=0xffff6f17, >> commit=<optimized out>) >> at /usr/src/contrib/jemalloc/include/jemalloc/internal/ehooks.h:208 >> #5 __je_extent_alloc_wrapper (tsdn=tsdn@entry=0x2a2e4060, pac=0x2a601810, >> ehooks=<optimized out>, new_addr=<optimized out>, size=12288, >> alignment=4096, zero=true, commit=0xffff6f77, >> growing_retained=<optimized out>) at jemalloc_extent.c:1003 >> #6 0x2a1916e0 in __je_ecache_alloc_grow (tsdn=<optimized out>, >> tsdn@entry=0x2a2e4060, pac=pac@entry=0x2a601810, >> ehooks=ehooks@entry=0x2a600080, ecache=<optimized out>, >> ecache@entry=0x2a603dd0, >> expand_edata=0x0, size=12288, alignment=4096, zero=<optimized out>, >> guarded=<optimized out>) at jemalloc_extent.c:126 >> #7 0x2a1c9680 in pac_alloc_real (tsdn=0x2a2e4060, pac=0x2a601810, >> ehooks=0x2a600080, size=12288, alignment=4096, zero=<optimized out>, >> guarded=false) at jemalloc_pac.c:124 >> #8 pac_alloc_impl (tsdn=tsdn@entry=0x2a2e4060, self=0x2a601810, >> size=size@entry=12288, alignment=4096, zero=<optimized out>, guarded=false, >> frequent_reuse=<optimized out>, >> deferred_work_generated=<optimized out>) at jemalloc_pac.c:178 >> #9 0x2a1c7ae8 in pai_alloc (tsdn=0x2a2e4060, self=0x0, size=12288, >> alignment=2147483615, zero=<optimized out>, guarded=false, >> frequent_reuse=true, deferred_work_generated=<optimized out>) >> at /usr/src/contrib/jemalloc/include/jemalloc/internal/pai.h:43 >> #10 __je_pa_alloc (tsdn=tsdn@entry=0x2a2e4060, shard=shard@entry=0x2a601800, >> size=12288, alignment=<optimized out>, slab=true, szind=25, zero=<optimized >> out>, guarded=false, >> deferred_work_generated=0xffff703f) at jemalloc_pa.c:139 >> #11 0x2a16b9f8 in arena_slab_alloc (tsdn=tsdn@entry=0x2a2e4060, >> arena=0x2a600540, binind=25, binshard=0, bin_info=0x2a21ff0c >> <__je_bin_infos+1200>) at jemalloc_arena.c:839 >> #12 0x2a16ac98 in __je_arena_cache_bin_fill_small (tsdn=0x2a2e4060, >> arena=0x2a600540, cache_bin=cache_bin@entry=0x2a2e4528, >> cache_bin_info=0x2a6004f2, binind=25, nfill=5) at jemalloc_arena.c:1034 >> #13 0x2a1b5694 in __je_tcache_alloc_small_hard (tsdn=0x0, >> tsdn@entry=0x2a2e4060, arena=0x0, arena@entry=0x2a600540, >> tcache=tcache@entry=0x2a2e42c8, cache_bin=cache_bin@entry=0x2a2e4528, >> binind=25, >> tcache_success=0xffff70ef) at jemalloc_tcache.c:238 >> #14 0x2a16cef4 in tcache_alloc_small (tsd=<optimized out>, arena=0x2a600540, >> tcache=0x2a2e42c8, size=<optimized out>, binind=25, zero=false, >> slow_path=true) >> at /usr/src/contrib/jemalloc/include/jemalloc/internal/tcache_inlines.h:68 >> #15 arena_malloc (tsdn=<optimized out>, arena=<optimized out>, size=1536, >> ind=25, zero=false, tcache=0x2a2e42c8, slow_path=true) >> at >> /usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:151 >> #16 0x2a16cb88 in __je_arena_palloc (tsdn=0x0, tsdn@entry=0x2a2e4060, >> arena=<optimized out>, usize=<optimized out>, usize@entry=1536, >> alignment=alignment@entry=8, zero=false, tcache=0x2a2e42c8) >> at jemalloc_arena.c:1224 >> #17 0x2a16559c in ipallocztm (tsdn=0x2a2e4060, tsdn@entry=0x2a2e42c8, >> usize=1536, alignment=8, zero=false, tcache=0x2a2e42c8, is_internal=false, >> arena=0x0) >> at >> /usr/src/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h:80 >> #18 ipalloct (tsdn=0x0, tsdn@entry=0x2a2e4060, usize=1536, alignment=8, >> zero=false, tcache=0x2a2e42c8, arena=0x0) >> at >> /usr/src/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h:91 >> #19 0x2a1651f4 in imalloc_no_sample (sopts=0xffff71e4, dopts=0xffff71c4, >> tsd=0x2a2e4060, size=1536, usize=1536, ind=<optimized out>) at >> jemalloc_jemalloc.c:2398 >> #20 imalloc_body (sopts=0xffff71e4, dopts=0xffff71c4, tsd=0x2a2e4060) at >> jemalloc_jemalloc.c:2577 >> #21 0x2a156188 in imalloc (sopts=sopts@entry=0xffff71e4, dopts=<optimized >> out>, dopts@entry=0xffff71c4) at jemalloc_jemalloc.c:2693 >> #22 0x2a15677c in __aligned_alloc (alignment=8, size=1536) at >> jemalloc_jemalloc.c:2821 >> #23 0x29e61a00 in std::__1::__libcpp_aligned_alloc[abi:se190107](unsigned >> int, unsigned int) (__alignment=8, __size=<optimized out>) >> at /usr/src/contrib/llvm-project/libcxx/include/__memory/aligned_alloc.h:43 >> #24 operator_new_aligned_impl (size=<optimized out>, alignment=8) at >> /usr/src/contrib/llvm-project/libcxx/src/new.cpp:129 >> #25 operator new (size=<optimized out>, alignment=<optimized out>) at >> /usr/src/contrib/llvm-project/libcxx/src/new.cpp:141 >> #26 0x2631dde0 in allocateBuckets () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:915 >> #27 grow () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:849 >> #28 0x2631dd0c in grow () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:580 >> #29 0x2631dd0c in InsertIntoBucketImpl<llvm::BasicBlock const*> () from >> /usr/lib/libprivatellvm.so.19 >> #30 0x2631daa4 in InsertIntoBucket<llvm::BasicBlock const* const&> () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:590 >> #31 FindAndConstruct () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:369 >> #32 0x2631d49c in operator[] () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/DenseMap.h:373 >> #33 analyzeBasicBlock () at >> /usr/src/contrib/llvm-project/llvm/lib/Analysis/CodeMetrics.cpp:234 >> #34 0x28e2c3ec in run () at >> /usr/src/contrib/llvm-project/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp:643 >> #35 0x28f4fd70 in runIPSCCP () at >> /usr/src/contrib/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp:165 >> #36 run () at >> /usr/src/contrib/llvm-project/llvm/lib/Transforms/IPO/SCCP.cpp:403 >> #37 0x27b85d14 in llvm::detail::PassModel<llvm::Module, llvm::IPSCCPPass, >> llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, >> llvm::AnalysisManager<llvm::Module>&) () >> at >> /usr/src/contrib/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:90 >> #38 0x276ee244 in run () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:81 >> #39 0x22174ffc in RunOptimizationPipeline () at >> /usr/src/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp:1114 >> #40 0x2216cfb8 in EmitAssembly () at >> /usr/src/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp:1179 >> #41 EmitBackendOutput () at >> /usr/src/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp:1341 >> #42 0x225cbca0 in HandleTranslationUnit () at >> /usr/src/contrib/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp:354 >> #43 0x22cff8e4 in ParseAST () at >> /usr/src/contrib/llvm-project/clang/lib/Parse/ParseAST.cpp:184 >> #44 0x22b5a7b8 in Execute () at >> /usr/src/contrib/llvm-project/clang/lib/Frontend/FrontendAction.cpp:1078 >> --Type <RET> for more, q to quit, c to continue without paging-- >> #45 0x22adb800 in ExecuteAction () at >> /usr/src/contrib/llvm-project/clang/lib/Frontend/CompilerInstance.cpp:1061 >> #46 0x22bf6a90 in ExecuteCompilerInvocation () at >> /usr/src/contrib/llvm-project/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp:280 >> #47 0x0002afc8 in cc1_main () at >> /usr/src/contrib/llvm-project/clang/tools/driver/cc1_main.cpp:284 >> #48 0x00038548 in ExecuteCC1Tool () at >> /usr/src/contrib/llvm-project/clang/tools/driver/driver.cpp:215 >> #49 0x227877ec in operator() () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/STLFunctionalExtras.h:68 >> #50 operator() () at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Job.cpp:440 >> #51 callback_fn<(lambda at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Job.cpp:440:22)>(void) () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/STLFunctionalExtras.h:45 >> #52 0x27d88624 in operator() () at >> /usr/src/contrib/llvm-project/llvm/include/llvm/ADT/STLFunctionalExtras.h:68 >> #53 RunSafely () at >> /usr/src/contrib/llvm-project/llvm/lib/Support/CrashRecoveryContext.cpp:426 >> #54 0x22786e90 in Execute () at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Job.cpp:440 >> #55 0x22748074 in ExecuteCommand () at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Compilation.cpp:199 >> #56 0x227483d0 in ExecuteJobs () at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Compilation.cpp:253 >> #57 0x22765bb8 in ExecuteCompilation () at >> /usr/src/contrib/llvm-project/clang/lib/Driver/Driver.cpp:1943 >> #58 0x00037ba4 in clang_main () at >> /usr/src/contrib/llvm-project/clang/tools/driver/driver.cpp:391 >> #59 0x000363a8 in main () at /usr/src/usr.bin/clang/clang/clang-driver.cpp:17 > > (gdb) list > 165 assert(size > 0); > 166 if (config_debug) { > 167 /* Check the whole first page. */ > 168 size_t *p = (size_t *)addr; > 169 for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { > 170 assert(p[i] == 0); > 171 } > > (gdb) x /1024x addr > 0x2b629000: 0x00000000 0x00000000 0x00000000 0x00000000 > 0x2b629010: 0x00000000 0x00000000 0x00000000 0x00000000 > 0x2b629020: 0x00000000 0x00000000 0x00000000 0x00000000 > . . . > 0x2b629c10: 0x00000000 0x00000000 0x00000000 0x00000000 > 0x2b629c20: 0x00000000 0x00000000 0x00000000 0x00000000 > 0x2b629c30: 0x00000000 0x00000000 0x00000000 0x00000000 > 0x2b629c40: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > 0x2b629c50: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > 0x2b629c60: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > . . . > 0x2b629fd0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > 0x2b629fe0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > 0x2b629ff0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a > (gdb) > > It has more 0x5a5a5a5a 's after that. The backtrace indicates 3 pages (12288 Bytes, 3072 size_t's) at the __je_pa_alloc call. So looking around the beginning and end: Beginning (and somewhat before): . . . 0x2b628fc0: 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0x2b628fd0: 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0x2b628fe0: 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0x2b628ff0: 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 0xa5a5a5a5 (gdb) x /1024x ((size_t*)addr) 0x2b629000: 0x00000000 0x00000000 0x00000000 0x00000000 0x2b629010: 0x00000000 0x00000000 0x00000000 0x00000000 0x2b629020: 0x00000000 0x00000000 0x00000000 0x00000000 . . . End (and just after): . . . 0x2b62bfd0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x2b62bfe0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x2b62bff0: 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a 0x5a5a5a5a (gdb) x /1024x ((size_t*)addr)+3072 0x2b62c000: Cannot access memory at address 0x2b62c000 For reference: static inline void * ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { bool orig_zero = *zero; void *ret; extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { ret = ehooks_default_alloc_impl(tsdn, new_addr, size, alignment, zero, commit, ehooks_ind_get(ehooks)); } else { ehooks_pre_reentrancy(tsdn); ret = extent_hooks->alloc(extent_hooks, new_addr, size, alignment, zero, commit, ehooks_ind_get(ehooks)); ehooks_post_reentrancy(tsdn); } assert(new_addr == NULL || ret == NULL || new_addr == ret); assert(!orig_zero || *zero); if (*zero && ret != NULL) { ehooks_debug_zero_check(ret, size); } return ret; } extent_hooks is optimized out so I do not know the status for the ehooks_default_alloc_impl vs. the tsd_pre_reentrancy and extent_hooks->alloc and ehooks_post_reentrancy usage that actually occurred. __je_extent_alloc_wrapper's context can be used to find that *zero was true when ehooks_alloc was called. It had to be true as of the ehooks_debug_zero_check call. Nothing suggests a smaller size was involved at any point. It looks like ehooks_default_alloc_impl or extent_hooks->alloc (whichever it was) just did not produce the correct content. Nothing yet stands out to me as looking likely to be somehow armv7 specific. Looking at: void * ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { arena_t *arena = arena_get(tsdn, arena_ind, false); /* NULL arena indicates arena_create. */ assert(arena != NULL || alignment == HUGEPAGE); dss_prec_t dss = (arena == NULL) ? dss_prec_disabled : (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED); void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, commit, dss); if (have_madvise_huge && ret) { pages_set_thp_state(ret, size); } return ret; } can get into atomic handling: atomic_load_u for ATOMIC_RELAXED is used. extent_alloc_dss vs. extent_alloc_mmap ends up involved via extent_alloc_core if ehooks_default_alloc_impl was used. I do not see that I can pull out much more based on my lack of familiarity. >> For reference: >> >> . . . >> Building >> /usr/obj/usr/src-investigation/arm.armv7/lib/clang/libclang/CodeGen/BackendUtil.pico >> . . . >> Building >> /usr/obj/usr/src-investigation/arm.armv7/lib/clang/libclang/CodeGen/CGDecl.pico >> <jemalloc>: >> /usr/src/contrib/jemalloc/include/jemalloc/internal/ehooks.h:170: Failed >> assertion: "p[i] == 0" >> . . . >> _ERROR_CMD='c++ -target armv7-gnueabihf-freebsd16.0 >> --sysroot=/usr/obj/usr/src-investigation/arm.armv7/tmp >> -B/usr/obj/usr/src-investigation/arm.armv7/tmp/usr/bin -fpic -DPIC -UPIC >> -O2 -pipe -fno-common >> -I/usr/obj/usr/src-investigation/arm.armv7/lib/clang/libclang >> -I/usr/obj/usr/src-investigation/arm.armv7/lib/clang/libllvm >> -I/usr/src-investigation/contrib/llvm-project/clang/lib/Basic >> -I/usr/src-investigation/contrib/llvm-project/clang/lib/Driver >> -I/usr/src-investigation/contrib/llvm-project/clang/lib/CodeGen >> -I/usr/src-investigation/contrib/llvm-project/clang/include >> -DCLANG_ENABLE_ARCMT -DCLANG_ENABLE_STATIC_ANALYZER >> -I/usr/src-investigation/lib/clang/include >> -I/usr/src-investigation/contrib/llvm-project/llvm/include >> -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS >> -DHAVE_VCS_VERSION_INC >> -DLLVM_DEFAULT_TARGET_TRIPLE=\"armv7-unknown-freebsd16.0-gnueabihf\" >> -DLLVM_HOST_TRIPLE=\"armv7-unknown-freebsd16.0\" -DDEFAULT_SYSROOT=\"\" >> -DLLVM_TARGET_ENABLE_AARCH64 -DLLVM_TARGET_ENABLE_ARM >> -DLLVM_TARGET_ENABLE_POWERPC -DLLVM_TARGET_ENABLE_RISCV >> -DLLVM_TARGET_ENABLE_X86 -DLLVM_NATIVE_ASMPARSER=LLVMInitializeARMAsmParser >> -DLLVM_NATIVE_ASMPRINTER=LLVMInitializeARMAsmPrinter >> -DLLVM_NATIVE_DISASSEMBLER=LLVMInitializeARMDisassembler >> -DLLVM_NATIVE_TARGET=LLVMInitializeARMTarget >> -DLLVM_NATIVE_TARGETINFO=LLVMInitializeARMTargetInfo >> -DLLVM_NATIVE_TARGETMC=LLVMInitializeARMTargetMC -ffunction-sections >> -fdata-sections -gline-tables-only -Wno-format-zero-length >> -fstack-protector-strong -Wdate-time -Wno-empty-body -Wno-string-plus-int >> -Wno-unused-const-variable -Wno-error=unused-but-set-parameter >> -Wno-error=cast-function-type-mismatch -Wno-tautological-compare >> -Wno-unused-value -Wno-parentheses-equality -Wno-unused-function >> -Wno-enum-conversion -Wno-unused-local-typedef -Wno-address-of-packed-member >> -Wno-switch -Wno-switch-enum -Wno-knr-promoted-parameter -Wno-parentheses >> -Qunused-arguments -fno-exceptions -fno-rtti -gline-tables-only -std=c++17 >> -stdlib=libc++ -c >> /usr/src-investigation/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp >> -o CodeGen/BackendUtil.pico;' === Mark Millard marklmi at yahoo.com
