I'm currently trying to set up a couple of icinga2 satalites on OpenBSD,
but they keep crashing after
/var/lib/icinga2/api/zones-stage/pub//_etc/generated_dbconfig_hosts.conf
has been copied to
/var/lib/icinga2/api/zones/pub//_etc/generated_dbconfig_hosts.conf.
I managed to get the following backtrace:
#1 0x00000a94dd2ce2ae in _libc_abort () at /usr/src/lib/libc/stdlib/abort.c:61
#2 0x00000a94dd231e9c in _libc_pthread_mutex_unlock (mutexp=<optimized out>)
at /usr/src/lib/libc/thread/rthread_mutex.c:265
#3 0x00000a9272acc510 in boost::posix::pthread_mutex_unlock (m=0xa927306f488
<icinga::ApiListener::m_ConfigSyncStageLock>) at
/usr/local/include/boost/thread/pthread/mutex.hpp:71
#4 boost::mutex::unlock (this=0xa927306f488
<icinga::ApiListener::m_ConfigSyncStageLock>) at
/usr/local/include/boost/thread/pthread/mutex.hpp:125
#5 boost::unique_lock<boost::mutex>::~unique_lock (this=<optimized out>) at
/usr/local/include/boost/thread/lock_types.hpp:331
#6 icinga::intrusive_ptr_release<boost::unique_lock<boost::mutex> >
(object=0xa952e854d60) at
/usr/ports/pobj/icinga2-2.11.5/icinga2-2.11.5/lib/base/shared.hpp:27
#7 boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> >
>::~intrusive_ptr (this=<optimized out>) at
/usr/local/include/boost/smart_ptr/intrusive_ptr.hpp:98
#8
icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex>
> > const&)::$_34::~$_34() (this=0xa949f7244c8)
at
/usr/ports/pobj/icinga2-2.11.5/icinga2-2.11.5/lib/remote/apilistener-filesync.cpp:648
#9
std::__1::__compressed_pair_elem<icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> > >
const&)::$_34, 0, false>::~__compressed_pair_elem() (this=0xa949f7244c8)
at /usr/include/c++/v1/memory:2134
#10 0x00000a9272acc423 in
std::__1::__function::__alloc_func<icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> > >
const&)::$_34,
std::__1::allocator<icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> > >
const&)::$_34>, void (icinga::ProcessResult const&)>::destroy() (this=<optimized
out>) at /usr/include/c++/v1/functional:1546
#11
std::__1::__function::__func<icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> > >
const&)::$_34,
std::__1::allocator<icinga::ApiListener::AsyncTryActivateZonesStage(std::__1::vector<icinga::String,
std::__1::allocator<icinga::String> > const&,
boost::intrusive_ptr<icinga::Shared<boost::unique_lock<boost::mutex> > >
const&)::$_34>, void (icinga::ProcessResult const&)>::destroy_deallocate()
(this=0xa949f7244c0) at /usr/include/c++/v1/functional:1643
#12 0x00000a92729bad13 in std::__1::__function::__value_func<void
(icinga::ProcessResult const&)>::~__value_func() (this=<optimized out>) at
/usr/include/c++/v1/functional:1758
#13 std::__1::function<void (icinga::ProcessResult const&)>::~function()
(this=<optimized out>) at /usr/include/c++/v1/functional:2334
#14 std::__1::__bind<std::__1::function<void (icinga::ProcessResult const&)>&,
icinga::ProcessResult&>::~__bind() (this=<optimized out>) at
/usr/include/c++/v1/functional:2648
#15 std::__1::__compressed_pair_elem<std::__1::__bind<std::__1::function<void
(icinga::ProcessResult const&)>&, icinga::ProcessResult&>, 0,
false>::~__compressed_pair_elem() (this=<optimized out>) at
/usr/include/c++/v1/memory:2134
#16 std::__1::__function::__alloc_func<std::__1::__bind<std::__1::function<void
(icinga::ProcessResult const&)>&, icinga::ProcessResult&>,
std::__1::allocator<std::__1::__bind<std::__1::function<void
(icinga::ProcessResult const&)>&, icinga::ProcessResult&> >, void
()>::destroy() (this=<optimized out>)
at /usr/include/c++/v1/functional:1546
#17 std::__1::__function::__func<std::__1::__bind<std::__1::function<void
(icinga::ProcessResult const&)>&, icinga::ProcessResult&>,
std::__1::allocator<std::__1::__bind<std::__1::function<void
(icinga::ProcessResult const&)>&, icinga::ProcessResult&> >, void
()>::destroy_deallocate() (this=0xa94b97a1000)
at /usr/include/c++/v1/functional:1643
#18 0x00000a92729d438d in std::__1::__function::__value_func<void
()>::~__value_func() (this=<optimized out>) at
/usr/include/c++/v1/functional:1758
#19 std::__1::function<void ()>::~function() (this=<optimized out>) at
/usr/include/c++/v1/functional:2334
#20 bool icinga::ThreadPool::Post<std::__1::function<void ()>
>(std::__1::function<void ()>,
icinga::SchedulerPolicy)::{lambda()#1}::~SchedulerPolicy() (this=<optimized
out>) at
/usr/ports/pobj/icinga2-2.11.5/icinga2-2.11.5/lib/base/threadpool.hpp:59
#21 boost::asio::system_executor::dispatch<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>, icinga::SchedulerPolicy)::{lambda()#1}, std::__1::allocator<void>
>(bool icinga::ThreadPool::Post<std::__1::function<void ()>
>>(std::__1::function<void ()>, icinga::SchedulerPolicy)::{lambda()#1}&&,
>std::__1::allocator<void> const&) const (this=<optimized out>,
f=<optimized out>) at /usr/local/include/boost/asio/impl/system_executor.hpp:40
#22 0x00000a92729d420c in boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>::operator()() (this=<optimized out>) at
/usr/local/include/boost/asio/detail/work_dispatcher.hpp:58
#23 boost::asio::asio_handler_invoke<boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>
>(boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>&, ...) (function=...) at
/usr/local/include/boost/asio/handler_invoke_hook.hpp:69
#24
boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>, bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>(boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>&, bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>, icinga::SchedulerPolicy)::{lambda()#1}&) (function=..., context=...)
at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:37
#25 boost::asio::detail::executor_op<boost::asio::detail::work_dispatcher<bool
icinga::ThreadPool::Post<std::__1::function<void ()> >(std::__1::function<void
()>,
icinga::SchedulerPolicy)::{lambda()#1}>, std::__1::allocator<void>,
boost::asio::detail::scheduler_operation>::do_complete(void*,
std::__1::allocator<void>*, boost::system::error_code const&, unsigned
long) (owner=0xa94be39e300, base=0xa94b97a1e00) at
/usr/local/include/boost/asio/detail/executor_op.hpp:70
#26 0x00000a927292c9c8 in boost::asio::detail::scheduler_operation::complete
(this=<optimized out>, owner=0xa94be39e300, ec=...,
bytes_transferred=<optimized out>) at
/usr/local/include/boost/asio/detail/scheduler_operation.hpp:40
#27 boost::asio::detail::scheduler::do_run_one (this=0xa94be39e300, lock=...,
this_thread=..., ec=...) at
/usr/local/include/boost/asio/detail/impl/scheduler.ipp:401
#28 0x00000a927292c492 in boost::asio::detail::scheduler::run
(this=0xa94be39e300, ec=...) at
/usr/local/include/boost/asio/detail/impl/scheduler.ipp:154
#29 0x00000a927293d867 in boost::asio::thread_pool::thread_function::operator()
(this=<optimized out>) at /usr/local/include/boost/asio/impl/thread_pool.ipp:33
#30
boost::asio::detail::posix_thread::func<boost::asio::thread_pool::thread_function>::run
(this=0xa9505e37ac0) at
/usr/local/include/boost/asio/detail/posix_thread.hpp:86
#31 0x00000a927293d7a5 in
boost::asio::detail::boost_asio_detail_posix_thread_function
(arg=0xa9505e37ac0) at
/usr/local/include/boost/asio/detail/impl/posix_thread.ipp:74
#32 0x00000a95076c10d1 in _rthread_start (v=<optimized out>) at
/usr/src/lib/librthread/rthread.c:96
#33 0x00000a94dd2c6c58 in __tfork_thread () at
/usr/src/lib/libc/arch/amd64/sys/tfork_thread.S:77
#34 0x0000000000000000 in ?? ()
And for shits and giggles I managed to confirm this by setting
_rthread_debug_level to 5, but it doesn't appear to give any
additional information:
icinga-checker9# grep -F 0xe0c6dd24f00 /tmp/mutex_debug
0xe0ca5effa40: mutex_lock 0xe0c6dd24f00 (0x0)
0xe0ca5effa40: mutex_unlock 0xe0c6dd24f00 (0xe0ca5effa40)
0xe0be5accc40: mutex_lock 0xe0c6dd24f00 (0x0)
0xe0ca5eff640: mutex_unlock 0xe0c6dd24f00 (0xe0be5accc40)
0xe0ca5eff640: different owner 0xe0c6dd24f00 (0xe0be5accc40)
Linux instances of the same icinga version don't seem to be affected.
I'm not familiar enough with icinga2/boost/C++ to know where I have to
take it from here. Any cluesticks out there?
martijn@
