Hi, all
rgw crash on thread notif-worker0 when using rgw persistent notification
function in ceph version: v16.2.13. I haven't find root cause.
Backtrace:
0> 2025-05-13T13:36:45.435+0000 7f0813c64700 -1 *** Caught signal (Aborted) **
in thread 7f0813c64700 thread_name:notif-worker0
1: /lib64/libpthread.so.0(+0x12cf0) [0x7f084607acf0]
2: gsignal()
3: abort()
4: /lib64/libstdc++.so.6(+0x9009b) [0x7f084506809b]
5: /lib64/libstdc++.so.6(+0x9654c) [0x7f084506e54c]
6: /lib64/libstdc++.so.6(+0x965a7) [0x7f084506e5a7]
7: /lib64/libstdc++.so.6(+0x96808) [0x7f084506e808]
8: (rgw::notify::Manager::Manager(ceph::common::CephContext*, unsigned int,
unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned
int, unsigned int, rgw::sal::RGWRadosStore*)::{lambda()#2}::operator()()
const+0x240) [0x7f0852c176c0]
9: /lib64/libstdc++.so.6(+0xc2b23) [0x7f084509ab23]
10: /lib64/libpthread.so.0(+0x81ca) [0x7f08460701ca]
11: clone()
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to
interpret this.
===================
Above coredump info had no symbol info , so I compile same version with
debuginfo. However, there are differen coredump info:
gdb) bt
#0 0x00007fd4edc6dacf in raise () from /lib64/libc.so.6
#1 0x00007fd4edc40ea5 in abort () from /lib64/libc.so.6
#2 0x00007fd4edc40d79 in __assert_fail_base.cold.0 () from /lib64/libc.so.6
#3 0x00007fd4edc66426 in __assert_fail () from /lib64/libc.so.6
#4 0x00007fd4fb81d7cd in
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>::allocate()
() from /lib64/libradosgw.so.2
#5 0x00007fd4fbe8491b in
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits> >
boost::context::callcc<boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
>::operator()()::{lambda(boost::context::continuation&&)#1}>(std::allocator_arg_t,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>&&,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::as
io::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
>::operator()()::{lambda(boost::context::continuation&&)#1}&&) () from
/lib64/libradosgw.so.2
#6 0x00007fd4fbe8554f in
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_t
ype> > >)#7}::stack_traits> >::operator()() () from /lib64/libradosgw.so.2
#7 0x00007fd4fbe85999 in void
boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type
const, spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yiel
d_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#7}::stack_traits> >&, std::allocator<void>
>(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl>
const&, boost::asio::io_context::executor_type const&,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#7}::stack_traits> >&, std::allocator<void> const&) () from
/lib64/libradosgw.so.2
#8 0x00007fd4fbe85c55 in
std::enable_if<spawn::detail::is_stack_allocator<std::decay<boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#7}::stack_traits> >::type, void>::value, void>::type
spawn::spawn<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<bo
ost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#7}::stack_traits>
>(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >,
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_cont
ext<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#7}&&,
std::decay&&) () from /lib64/libradosgw.so.2
#9 0x00007fd4fbe888fe in
rgw::notify::Manager::process_queue(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >) () from
/lib64/libradosgw.so.2
#10 0x00007fd4fbe8c711 in
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#6}::operator()(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> > >) const
() from /lib64/libradosgw.so.2
#11 0x00007fd4fbe8cab4 in
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#6}::stack_traits>
>::operator()()::{lambda(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::str
and<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#6}::continuation&&)#1}::operator()(rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >
>)#6}::continuation) const () from /lib64/libradosgw.so.2
#12 0x00007fd4fbe8cc90 in
boost::context::detail::record<boost::context::continuation,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_context<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
>::operator()()::{lambda(boost::context::continuation&&)#1}>::run(void*) ()
from /lib64/libradosgw.so.2
#13 0x00007fd4fbe8cd58 in void
boost::context::detail::context_entry<boost::context::detail::record<boost::context::continuation,
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>,
spawn::detail::spawn_helper<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> >,
rgw::notify::Manager::process_queues(spawn::basic_yield_context<boost::asio::executor_binder<void
(*)(), boost::asio::strand<boost::asio::io_context::executor_type> >
>)::{lambda(spawn::basic_yield_c--Type <RET> for more, q to quit, c to continue
without paging--
ontext<boost::asio::executor_binder<void (*)(),
boost::asio::strand<boost::asio::io_context::executor_type> > >)#6},
boost::context::basic_protected_fixedsize_stack<boost::context::stack_traits>
>::operator()()::{lambda(boost::context::continuation&&)#1}>
>(boost::context::detail::transfer_t) () from /lib64/libradosgw.so.2
#14 0x00007fd4fbf5e94f in make_fcontext () from /lib64/libradosgw.so.2
#15 0x0000000000000000 in ?? ()
I have found a similar issue in https://github.com/ceph/ceph/pull/39521, but it
haven't reslove it.
_______________________________________________
ceph-users mailing list -- [email protected]
To unsubscribe send an email to [email protected]
