On 7.7.2021. 12:46, Hrvoje Popovski wrote: > Panic can be triggered when i have parallel diff and sending traffic > over ipsec tunnel and on other side while traffic is flowing i'm > restarting isakmpd daemon and while negotiating ipsec doing ifconfig ix1 > down && ifconfig ix1 up ... sometimes it panic all by itself.. it's not > that i know exactly what's going on ..
Different panic on same setup ...
uvm_fault(0xffffffff8221de20, 0x8, 0, 1) -> e kernel: page fault trap, code=0 Stopped at pool_get+0x6e: movq 0x8(%rax),%r11 TID PID UID PRFLAGS PFLAGS CPU COMMAND 288284 51828 0 0x3 0 0 ifconfig 329904 31282 0 0x14000 0x200 4 softnet 157417 71744 0 0x14000 0x200 2 softnet *152664 84791 0 0x14000 0x200 3K softnet 58812 4615 0 0x14000 0x200 1 softnet pool_get(ffffffff82153918,a) at pool_get+0x6e ipsp_acquire_sa(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,ffff80002386b068,ffffff84) at ipsp_acquire_sa+0x124 ipsp_spd_lookup(fffffd80b842db00,2,14,ffff80002386b24c,2,0) at ipsp_spd_lookup+0xb4e ip_output_ipsec_lookup(fffffd80b842db00,14,ffff80002386b24c,0,0) at ip_output_ipsec_lookup+0x4d ip_output(fffffd80b842db00,0,ffff80002386b3d8,1,0,0) at ip_output+0x42a ip_forward(fffffd80b842db00,ffff800000087048,fffffd83b3fecb68,0) at ip_forward+0x26a ip_input_if(ffff80002386b518,ffff80002386b524,4,0,ffff800000087048) at ip_input_if+0x365 ipv4_input(ffff800000087048,fffffd80b842db00) at ipv4_input+0x39 if_input_process(ffff800000087048,ffff80002386b598) at if_input_process+0x6f ifiq_process(ffff800000086d00) at ifiq_process+0x69 taskq_thread(ffff800000030100) at taskq_thread+0x9f end trace frame: 0x0, count: 4 https://www.openbsd.org/ddb.html describes the minimum info required in bug reports. Insufficient info makes it difficult to find and fix bugs. ddb{3}> show reg rdi 0xffffffff82153918 ipsec_acquire_pool rsi 0xa rbp 0xffff80002386af70 rbx 0 rdx 0xfffffd842ae97650 rcx 0x1 rax 0 r8 0xffffff84 r9 0xffff80000016a604 r10 0xffff8000012c7d40 r11 0x4f581b7438ceb9e9 r12 0xfffffd842ae9766c r13 0xfffffd842ae97580 r14 0xffffffff82153918 ipsec_acquire_pool r15 0xa rip 0xffffffff8133e97e pool_get+0x6e cs 0x8 rflags 0x10246 __ALIGN_SIZE+0xf246 rsp 0xffff80002386aed0 ss 0 pool_get+0x6e: movq 0x8(%rax),%r11 ddb{3}> ddb{3}> ps PID TID PPID UID S FLAGS WAIT COMMAND 51828 288284 1009 0 7 0x3 ifconfig 1009 461325 1 0 3 0x10008b sigsusp ksh 56847 290195 1 0 3 0x100098 poll cron 83452 510449 26068 95 3 0x100092 kqread smtpd 41700 201644 26068 103 3 0x100092 kqread smtpd 26028 130559 26068 95 3 0x100092 kqread smtpd 11949 518693 26068 95 3 0x100092 kqread smtpd 4520 211282 26068 95 3 0x100092 kqread smtpd 82047 205282 26068 95 3 0x100092 kqread smtpd 26068 82309 1 0 3 0x100080 kqread smtpd 46745 316858 1 0 3 0x88 select sshd 5198 371736 71443 68 3 0x90 select isakmpd 71443 13600 1 0 3 0x80 netio isakmpd 71184 265525 1 0 3 0x100080 poll ntpd 26181 322162 3615 83 3 0x100092 poll ntpd 3615 143908 1 83 3 0x100092 poll ntpd 82091 209375 43348 73 3 0x100090 kqread syslogd 43348 363890 1 0 3 0x100082 netio syslogd 16351 58282 0 0 3 0x14200 bored smr 27233 372715 0 0 3 0x14200 pgzero zerothread 68321 430498 0 0 3 0x14200 aiodoned aiodoned 10629 99914 0 0 3 0x14200 syncer update 74747 455079 0 0 3 0x14200 cleaner cleaner 67178 305272 0 0 3 0x14200 reaper reaper 54113 244787 0 0 3 0x14200 pgdaemon pagedaemon 88516 52910 0 0 3 0x14200 bored crynlk 84865 523006 0 0 3 0x14200 bored crypto 87886 398410 0 0 3 0x14200 usbtsk usbtask 79482 516951 0 0 3 0x14200 usbatsk usbatsk 7554 475208 0 0 3 0x40014200 acpi0 acpi0 55853 38718 0 0 7 0x40014200 idle5 13965 462123 0 0 3 0x40014200 idle4 14774 279411 0 0 3 0x40014200 idle3 25764 508246 0 0 3 0x40014200 idle2 14959 77683 0 0 3 0x40014200 idle1 95869 101965 0 0 3 0x14200 bored sensors 31282 329904 0 0 7 0x14200 softnet 71744 157417 0 0 7 0x14200 softnet *84791 152664 0 0 7 0x14200 softnet 4615 58812 0 0 7 0x14200 softnet 45524 379795 0 0 3 0x14200 bored systqmp 11914 16163 0 0 3 0x14200 bored systq 48612 387270 0 0 3 0x40014200 netlock softclock 64200 21926 0 0 3 0x40014200 idle0 1 236011 0 0 3 0x82 wait init 0 0 -1 0 3 0x10200 scheduler swapper ddb{3}> ddb{3}> trace /t 0t288284 __kernel_virt_to_phys() at __kernel_virt_to_phys+0x1ddc045 end of kernel end trace frame: 0x1d717518000, count: -1 ddb{3}> ddb{3}> trace /t 0t329904 fffffd80b842d800(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638 ) at 0xfffffd80b842d800 Bad frame pointer: 0xffff800000087000 end trace frame: 0xffff800000087000, count: -1 ddb{3}> trace /t 0t157417 fffffd80bab65800(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638 ) at 0xfffffd80bab65800 Bad frame pointer: 0xffff800000087000 end trace frame: 0xffff800000087000, count: -1 ddb{3}> trace /t 0t152664 fffffd80b842db00(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638 ) at 0xfffffd80b842db00 Bad frame pointer: 0xffff800000087000 end trace frame: 0xffff800000087000, count: -1 ddb{3}> ddb{3}> trace /t 0t58812 fffffd80bb6bd000(200000001,ffff800000080600,0,0,31786900000001,ffffffff82118638 ) at 0xfffffd80bb6bd000 Bad frame pointer: 0xffff800000087000 end trace frame: 0xffff800000087000, count: -1 ddb{3}> ddb{3}> mach ddbcpu 0 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffffffff82110ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82230fb0) at __mp_lock+0xb3 softintr_dispatch(0) at softintr_dispatch+0x4a Xsoftclock() at Xsoftclock+0x1f __mp_lock(ffffffff82230fb0) at __mp_lock+0xb3 syscall(ffff800023915380) at syscall+0x2a4 Xsyscall() at Xsyscall+0x128 end of kernel end trace frame: 0x7f7ffffe98e0, count: 6 ddb{0}> ddb{0}> mach ddbcpu 1 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff800022409ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82230fb0) at __mp_lock+0xa7 esp_output(fffffd80bb6bd000,ffff8000012ef360,0,14,9) at esp_output+0x8c3 ipsp_process_packet(fffffd80bb6bd000,ffff8000012ef360,2,0) at ipsp_process_packet+0x466 ip_output_ipsec_send(ffff8000012ef360,fffffd80bb6bd000,ffff800023865548,1) at ip_output_ipsec_send+0x171 ip_output(fffffd80bb6bd000,0,ffff800023865548,1,0,0) at ip_output+0x8cd ip_forward(fffffd80bb6bd000,ffff800000087048,fffffd83b3fecb68,0) at ip_forward+0x26a ip_input_if(ffff800023865688,ffff800023865694,4,0,ffff800000087048) at ip_input_if+0x365 ipv4_input(ffff800000087048,fffffd80bb6bd000) at ipv4_input+0x39 if_input_process(ffff800000087048,ffff800023865708) at if_input_process+0x6f ifiq_process(ffff800000086c00) at ifiq_process+0x69 taskq_thread(ffff800000030000) at taskq_thread+0x9f end trace frame: 0x0, count: 1 ddb{1}> ddb{1}> mach ddbcpu 2 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff800022412ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82230fb0) at __mp_lock+0xac selwakeup(fffffd83b0d0d4e0) at selwakeup+0x11 sorwakeup(fffffd83b0d0d3c8) at sorwakeup+0x77 pfkeyv2_sendmessage(ffff800023870bb0,2,0,2,0,0) at pfkeyv2_sendmessage+0x2e1 pfkeyv2_acquire(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,fffffd8386f2 fe8c,ffff800023870db8) at pfkeyv2_acquire+0x54f ipsp_spd_lookup(fffffd80bab65800,2,14,ffff800023870f9c,2,0) at ipsp_spd_lookup+0xb4e ip_output_ipsec_lookup(fffffd80bab65800,14,ffff800023870f9c,0,0) at ip_output_ipsec_lookup+0x4d ip_output(fffffd80bab65800,0,ffff800023871128,1,0,0) at ip_output+0x42a ip_forward(fffffd80bab65800,ffff800000087048,fffffd83b3fecb68,0) at ip_forward+0x26a ip_input_if(ffff800023871268,ffff800023871274,4,0,ffff800000087048) at ip_input_if+0x365 ipv4_input(ffff800000087048,fffffd80bab65800) at ipv4_input+0x39 end trace frame: 0xffff8000238712d0, count: 0 ddb{2}> ddb{2}> mach ddbcpu 3 Stopped at pool_get+0x6e: movq 0x8(%rax),%r11 pool_get(ffffffff82153918,a) at pool_get+0x6e ipsp_acquire_sa(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,ffff80002386b068,ffffff84) at ipsp_acquire_sa+0x124 ipsp_spd_lookup(fffffd80b842db00,2,14,ffff80002386b24c,2,0) at ipsp_spd_lookup+0xb4e ip_output_ipsec_lookup(fffffd80b842db00,14,ffff80002386b24c,0,0) at ip_output_ipsec_lookup+0x4d ip_output(fffffd80b842db00,0,ffff80002386b3d8,1,0,0) at ip_output+0x42a ip_forward(fffffd80b842db00,ffff800000087048,fffffd83b3fecb68,0) at ip_forward+0x26a ip_input_if(ffff80002386b518,ffff80002386b524,4,0,ffff800000087048) at ip_input_if+0x365 ipv4_input(ffff800000087048,fffffd80b842db00) at ipv4_input+0x39 if_input_process(ffff800000087048,ffff80002386b598) at if_input_process+0x6f ifiq_process(ffff800000086d00) at ifiq_process+0x69 taskq_thread(ffff800000030100) at taskq_thread+0x9f end trace frame: 0x0, count: 4 ddb{3}> ddb{3}> mach ddbcpu 4 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff800022424ff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 __mp_lock(ffffffff82230fb0) at __mp_lock+0xb3 selwakeup(fffffd83b0d0d4e0) at selwakeup+0x11 sorwakeup(fffffd83b0d0d3c8) at sorwakeup+0x77 pfkeyv2_sendmessage(ffff800023876d20,2,0,2,0,0) at pfkeyv2_sendmessage+0x2e1 pfkeyv2_acquire(fffffd842ae97580,fffffd842ae9766c,fffffd842ae97650,fffffd8386f2 fd84,ffff800023876f28) at pfkeyv2_acquire+0x54f ipsp_spd_lookup(fffffd80b842d800,2,14,ffff80002387710c,2,0) at ipsp_spd_lookup+0xb4e ip_output_ipsec_lookup(fffffd80b842d800,14,ffff80002387710c,0,0) at ip_output_ipsec_lookup+0x4d ip_output(fffffd80b842d800,0,ffff800023877298,1,0,0) at ip_output+0x42a ip_forward(fffffd80b842d800,ffff800000087048,fffffd83b3fecb68,0) at ip_forward+0x26a ip_input_if(ffff8000238773d8,ffff8000238773e4,4,0,ffff800000087048) at ip_input_if+0x365 ipv4_input(ffff800000087048,fffffd80b842d800) at ipv4_input+0x39 end trace frame: 0xffff800023877440, count: 0 ddb{4}> ddb{4}> mach ddbcpu 5 Stopped at x86_ipi_db+0x12: leave x86_ipi_db(ffff80002242dff0) at x86_ipi_db+0x12 x86_ipi_handler() at x86_ipi_handler+0x80 Xresume_lapic_ipi() at Xresume_lapic_ipi+0x23 acpicpu_idle() at acpicpu_idle+0x281 sched_idle(ffff80002242dff0) at sched_idle+0x27e end trace frame: 0x0, count: 10 ddb{5}> ddb{3}> show all pool Name Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle ipsec acquire 264 2 0 0 2 0 2 2 0 8 1 tdb 1096 9 0 7 1 0 1 1 0 8 0 plcache 128 66 0 0 3 0 3 3 0 8 0 rtpcb 120 16 0 16 1 1 0 1 0 8 0 rtentry 112 54 0 12 2 0 2 2 0 8 0 unpcb 120 65 0 17 2 0 2 2 0 8 0 tcpcb 736 9 0 4 1 0 1 1 0 8 0 arp 120 19 0 12 1 0 1 1 0 8 0 inpcb 304 160 0 140 2 0 2 2 0 8 0 nd6 48 3 0 0 1 0 1 1 0 8 0 ipsec policy 352 4 0 2 1 0 1 1 0 8 0 pkpcb 40 2 0 1 1 0 1 1 0 8 0 cryptop 352 1146235 0 1146234 79970 48548 31422 79970 0 8 31421 art_heap8 4096 1 0 0 1 0 1 1 0 8 0 art_heap4 256 175 0 1 11 0 11 11 0 8 0 art_table 32 176 0 1 2 0 2 2 0 8 0 art_node 16 54 0 12 1 0 1 1 0 8 0 dirhash 1024 84 0 40 6 0 6 6 0 8 0 newdirblk 32 16 0 16 1 1 0 1 0 8 0 dirrem 64 1643 0 1643 26 26 0 26 0 8 0 mkdir 56 16 0 16 1 1 0 1 0 8 0 diradd 56 1652 0 1652 23 23 0 23 0 8 0 freefile 48 1624 0 1624 22 22 0 20 0 8 0 freeblks 192 1645 0 1645 82 82 0 81 0 8 0 freefrag 64 16 0 16 2 2 0 1 0 8 0 allocindir 104 10863 0 10863 239 239 0 208 0 8 0 indirdep 56 19 0 19 1 1 0 1 0 8 0 allocdir 128 2741 0 2741 76 76 0 76 0 8 0 bmsafemap 64 34 0 34 1 1 0 1 0 8 0 newblk 64 13604 0 13604 4 4 0 1 0 8 0 inodedep 160 1688 0 1688 70 70 0 70 0 8 0 pagedep 128 27 0 27 1 1 0 1 0 8 0 dino2pl 256 5101 0 1650 224 7 217 217 0 8 0 ffsino 272 5101 0 1650 238 7 231 231 0 8 0 nchpl 144 5411 0 2209 124 5 119 119 0 8 0 rtmask 32 2 0 1 1 0 1 1 0 8 0 uvmvnodes 72 5118 0 0 94 0 94 94 0 8 0 vnodes 224 5118 0 0 302 0 302 302 0 8 0 namei 1024 17535 0 17535 12 11 1 1 0 8 1 percpumem 48 32 0 0 1 0 1 1 0 8 0 aesni 576 6 0 4 1 0 1 1 1 8 0 ehcixfer 296 184 0 179 1 0 1 1 0 8 0 scxspl 216 89562 0 89562 16 15 1 2 0 8 1 plimitpl 152 25 0 12 1 0 1 1 0 8 0 sigapl 424 407 0 362 7 1 6 7 0 8 0 futexpl 56 4705 0 4705 2 2 0 1 0 8 0 knotepl 112 48 0 0 2 0 2 2 0 8 0 kqueuepl 216 8 0 0 1 0 1 1 0 8 0 pipepl 336 94 0 94 4 4 0 1 0 8 0 fdescpl 496 382 0 362 4 0 4 4 0 8 0 filepl 152 7094 0 6994 5 0 5 5 0 8 0 lockfpl 104 4 0 4 1 1 0 1 0 8 0 lockfspl 48 2 0 2 1 1 0 1 0 8 0 sessionpl 144 11 0 1 1 0 1 1 0 8 0 pgrppl 48 31 0 20 1 0 1 1 0 8 0 ucredpl 96 75 0 56 1 0 1 1 0 8 0 zombiepl 144 363 0 363 9 8 1 1 0 8 1 processpl 1080 407 0 362 5 1 4 4 0 8 0 procpl 672 438 0 393 6 1 5 6 0 8 0 sockpl 480 243 0 174 10 1 9 10 0 8 0 mcl4k 4096 1 0 0 1 0 1 1 0 8 0 mcl2k2 2112 879900 39 0 58661 1 58660 58660 0 8 0 mcl2k 2048 5 0 0 1 0 1 1 0 8 0 mtagpl 96 1736 0 0 43 0 43 43 0 8 0 mbufpl 256 879932 0 0 54997 2 54995 54995 0 8 0 bufpl 280 103327 0 64461 2778 1 2777 2777 0 8 0 anonpl 24 152833 0 146253 272 225 47 241 0 1513 0 amapchunkpl 152 10595 0 10161 112 92 20 104 0 158 0 amappl16 200 909 0 902 27 26 1 15 0 8 0 amappl15 192 152 0 140 2 1 1 1 0 8 0 amappl14 184 26 0 26 5 5 0 1 0 8 0 amappl13 176 27 0 26 1 0 1 1 0 8 0 amappl12 168 77 0 77 6 6 0 3 0 8 0 amappl11 160 120 0 86 3 1 2 2 0 8 0 amappl10 152 22 0 22 2 2 0 1 0 8 0 amappl9 144 20 0 20 5 5 0 1 0 8 0 amappl8 136 1114 0 1100 21 20 1 13 0 8 0 amappl7 128 119 0 116 2 1 1 2 0 8 0 amappl6 120 340 0 310 8 7 1 4 0 8 0 amappl5 112 217 0 199 3 2 1 3 0 8 0 amappl4 104 2223 0 2170 29 26 3 23 0 8 0 amappl3 96 689 0 663 9 8 1 7 0 8 0 amappl2 88 2994 0 2843 41 36 5 29 0 8 0 amappl1 80 10164 0 9465 21 4 17 19 0 8 0 amappl 88 4095 0 3925 22 16 6 21 0 92 0 dma8192 8192 1 0 1 1 1 0 1 0 8 0 dma4096 4096 7 0 1 1 0 1 1 0 8 0 dma2048 2048 76 0 76 37 36 1 1 0 8 1 dma1024 1024 74 0 74 37 36 1 1 0 8 1 dma512 512 47 0 46 1 0 1 1 0 8 0 dma256 256 7 0 7 1 1 0 1 0 8 0 dma128 128 17 0 17 1 1 0 1 0 8 0 dma64 64 2 0 2 1 1 0 1 0 8 0 dma32 32 9 0 9 1 1 0 1 0 8 0 dma16 16 10 0 10 3 3 0 1 0 8 0 aobjpl 64 2 0 0 1 0 1 1 0 8 0 uaddrrnd 24 382 0 362 1 0 1 1 0 8 0 uaddrbest 32 2 0 0 1 0 1 1 0 8 0 uaddr 24 382 0 362 1 0 1 1 0 8 0 vmmpekpl 168 98569 0 98546 2 0 2 2 0 8 0 vmmpepl 168 49045 0 47276 350 259 91 328 0 357 0 vmsppl 368 381 0 362 3 0 3 3 0 8 0 rwobjpl 56 17758 0 16585 84 63 21 70 0 8 0 pdppl 4096 771 0 724 92 45 47 69 0 8 0 pvpl 32 737117 0 723805 1309 1161 148 1267 0 265 23 pmappl 224 381 0 362 2 0 2 2 0 8 0 extentpl 40 179 0 114 1 0 1 1 0 8 0 phpool 112 55669 0 150 1589 2 1587 1587 0 8 0 ddb{3}>