***************** crashinfo --stacksummary ***************** /exports/testreports/68568/testresults/sanity3-zfs-rocky8.10_x86_64-rocky8.10_x86_64-retry1/oleg329-client-vmcore (4.18.0rh8.10-debug) ------- 27 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork youngest=4888s(pid=669), oldest=4908s(pid=3) ........................ acpi_thermal_pm 1 times ata_sff 1 times blkcg_punt_bio 1 times crypto 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kintegrityd 1 times knbd0-recv 1 times kstrp 1 times kthrotld 1 times md 1 times md_bitmap 1 times mm_percpu_wq 1 times netns 1 times nfsiod 1 times rcu_gp 1 times rcu_par_gp 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times slub_flushwq 1 times tpm_dev_wq 1 times writeback 1 times xprtiod 1 times zswap-shrink 1 times ------- 23 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork youngest=0s(pid=1365), oldest=4889s(pid=659) ........................ kworker/0:0 1 times kworker/0:1 1 times kworker/0:1H 1 times kworker/0:2 1 times kworker/0:2H 1 times kworker/1:0 1 times kworker/1:0H 1 times kworker/1:1 1 times kworker/1:2H 1 times kworker/1:3 1 times kworker/2:1 1 times kworker/2:1H 1 times kworker/2:2 1 times kworker/2:2H 1 times kworker/3:0 1 times kworker/3:0H 1 times kworker/3:2 1 times kworker/3:2H 1 times kworker/3:3 1 times kworker/u8:0 1 times kworker/u8:1 1 times kworker/u8:4 1 times kworker/u9:1 1 times ------- 16 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork youngest=0s(pid=18), oldest=4879s(pid=23) ........................ cpuhp/0 1 times cpuhp/1 1 times cpuhp/2 1 times cpuhp/3 1 times ksoftirqd/0 1 times ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 14 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_poll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=1s(pid=1010), oldest=4871s(pid=1014) ........................ NetworkManager 1 times gdbus 2 times gmain 3 times irqbalance 1 times polkitd 2 times rngd 1 times rpcbind 1 times sshd 1 times tuned 2 times ------- 13 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork youngest=0s(pid=144546), oldest=13s(pid=124816) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ldlm_bl_04 1 times ldlm_bl_05 1 times ldlm_bl_06 1 times ldlm_bl_07 1 times ldlm_bl_08 1 times ldlm_bl_09 1 times ldlm_bl_10 1 times ldlm_bl_11 1 times ldlm_bl_12 1 times ldlm_bl_13 1 times ------- 13 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork youngest=46s(pid=143766), oldest=792s(pid=124810) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cb00_002 1 times ldlm_cb00_003 1 times ldlm_cb01_000 1 times ldlm_cb01_001 1 times ldlm_cb01_002 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn01_000 1 times ldlm_cn01_001 1 times ldlm_cn01_002 1 times ------- 10 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 __x64_sys_futex #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=173s(pid=1163), oldest=4869s(pid=1044) ........................ JS Helper 4 times gssproxy 5 times rs:main Q:Reg 1 times ------- 10 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 kern_select #8 __x64_sys_select #9 do_syscall_64 #10 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1351), oldest=4865s(pid=1062) ........................ agetty 3 times dhclient 1 times fio 2 times sshd 2 times tuned 2 times ------- 7 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 do_epoll_wait #6 __x64_sys_epoll_wait #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=89s(pid=1), oldest=4869s(pid=1043) ........................ dbus-daemon 1 times gssproxy 1 times systemd 2 times systemd-journal 1 times systemd-logind 1 times systemd-udevd 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork youngest=4s(pid=2359), oldest=362s(pid=2355) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_01_00 1 times ptlrpcd_01_01 1 times ptlrpcd_rcv 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork youngest=5s(pid=2348), oldest=4210s(pid=2350) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd01_00 1 times socknal_sd01_01 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork youngest=3671s(pid=2346), oldest=4217s(pid=2343) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 cl_sync_io_wait #3 cl_sync_io_wait_recycle #4 ll_file_io_generic #5 do_file_write_iter #6 ll_file_write_iter #7 new_sync_write #8 __vfs_write #9 vfs_write #10 ksys_pwrite64 #11 __x64_sys_pwrite64 #12 do_syscall_64 #13 entry_SYSCALL_64_after_hwframe youngest=52s(pid=144537), oldest=52s(pid=144533) ........................ fio 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 do_wait #3 kernel_wait4 #4 __do_sys_wait4 #5 __x64_sys_wait4 #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=167s(pid=144378), oldest=4617s(pid=1551) ........................ auster 1 times bash 2 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 do_nanosleep #3 hrtimer_nanosleep #4 __x64_sys_nanosleep #5 do_syscall_64 #6 entry_SYSCALL_64_after_hwframe youngest=0s(pid=144526), oldest=58s(pid=1055) ........................ crond 1 times fio 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork youngest=4810s(pid=2341), oldest=4810s(pid=2342) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr01_000 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork youngest=4898s(pid=474), oldest=4898s(pid=467) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_tasks_kthread #3 kthread #4 ret_from_fork youngest=4908s(pid=11), oldest=4908s(pid=10) ........................ rcu_tasks_rude_ 1 times rcu_tasks_trace 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule_idle #2 do_idle #3 cpu_startup_entry #4 start_secondary #5 secondary_startup_64_no_verify youngest=4908s(pid=0), oldest=4908s(pid=0) ........................ swapper/2 1 times swapper/3 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_preempt_disabled #3 kthread #4 ret_from_fork youngest=0s(pid=144570), oldest=0s(pid=144570) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #0 crash_nmi_callback #1 nmi_handle #2 default_do_nmi #3 do_nmi #4 end_repeat_nmi #-1 native_safe_halt #5 native_safe_halt #6 kvm_wait #7 __pv_queued_spin_lock_slowpath #8 do_raw_spin_lock #9 _raw_spin_lock #10 osc_enter_cache #11 osc_queue_async_io #12 osc_page_cache_add #13 osc_io_commit_async #14 cl_io_commit_async #15 lov_io_commit_async #16 cl_io_commit_async #17 vvp_io_write_commit #18 vvp_io_write_start #19 cl_io_start #20 cl_io_loop #21 ll_file_io_generic #22 do_file_write_iter #23 ll_file_write_iter #24 new_sync_write #25 __vfs_write #26 vfs_write #27 ksys_pwrite64 #28 __x64_sys_pwrite64 #29 do_syscall_64 #30 entry_SYSCALL_64_after_hwframe youngest=0s(pid=144538), oldest=0s(pid=144538) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 cl_sync_io_wait #3 cl_sync_io_wait_recycle #4 ll_file_io_generic #5 do_file_read_iter #6 ll_file_read_iter #7 new_sync_read #8 __vfs_read #9 vfs_read #10 ksys_pread64 #11 __x64_sys_pread64 #12 do_syscall_64 #13 entry_SYSCALL_64_after_hwframe youngest=52s(pid=144536), oldest=52s(pid=144536) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 machine_kexec #1 __crash_kexec #2 crash_kexec #3 oops_end #4 no_context.cold.27 #5 __bad_area_nosemaphore #6 bad_area_nosemaphore #7 do_user_addr_fault #8 __do_page_fault #9 do_page_fault #10 page_fault #-1 __osc_extent_tree_dump #11 osc_enter_cache #12 osc_queue_async_io #13 osc_page_cache_add #14 osc_io_commit_async #15 cl_io_commit_async #16 lov_io_commit_async #17 cl_io_commit_async #18 vvp_io_write_commit #19 vvp_io_write_start #20 cl_io_start #21 cl_io_loop #22 ll_file_io_generic #23 do_file_write_iter #24 ll_file_write_iter #25 new_sync_write #26 __vfs_write #27 vfs_write #28 ksys_pwrite64 #29 __x64_sys_pwrite64 #30 do_syscall_64 #31 entry_SYSCALL_64_after_hwframe youngest=0s(pid=144535), oldest=0s(pid=144535) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 cl_sync_io_wait #3 cl_io_submit_sync #4 vvp_io_commit_sync #5 vvp_io_write_commit #6 vvp_io_write_start #7 cl_io_start #8 cl_io_loop #9 ll_file_io_generic #10 do_file_write_iter #11 ll_file_write_iter #12 new_sync_write #13 __vfs_write #14 vfs_write #15 ksys_pwrite64 #16 __x64_sys_pwrite64 #17 do_syscall_64 #18 entry_SYSCALL_64_after_hwframe youngest=0s(pid=144534), oldest=0s(pid=144534) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_completion_ast #3 ldlm_cli_enqueue_fini #4 ldlm_cli_enqueue #5 osc_enqueue_base #6 osc_lock_enqueue #7 cl_lock_enqueue #8 lov_lock_enqueue #9 cl_lock_enqueue #10 cl_lock_request #11 cl_io_lock #12 cl_io_loop #13 ll_file_io_generic #14 do_file_write_iter #15 ll_file_write_iter #16 new_sync_write #17 __vfs_write #18 vfs_write #19 ksys_pwrite64 #20 __x64_sys_pwrite64 #21 do_syscall_64 #22 entry_SYSCALL_64_after_hwframe youngest=52s(pid=144531), oldest=52s(pid=144531) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 pipe_wait #3 pipe_read #4 new_sync_read #5 __vfs_read #6 vfs_read #7 ksys_read #8 __x64_sys_read #9 do_syscall_64 #10 entry_SYSCALL_64_after_hwframe youngest=158s(pid=144380), oldest=158s(pid=144380) ........................ tee 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_killable #4 __kthread_create_on_node #5 kthread_create_on_node #6 nfs4_schedule_state_manager #7 nfs_expire_unreferenced_delegations #8 nfs4_renew_state #9 process_one_work #10 worker_thread #11 kthread #12 ret_from_fork youngest=0s(pid=137059), oldest=0s(pid=137059) ........................ kworker/0:3 1 times ------- 1 stacks like that: ---------- #0 crash_nmi_callback #1 nmi_handle #2 default_do_nmi #3 do_nmi #4 end_repeat_nmi #-1 __pv_queued_spin_lock_slowpath #5 __pv_queued_spin_lock_slowpath #6 do_raw_spin_lock #7 _raw_spin_lock #8 osc_announce_cached #9 osc_brw_prep_request #10 osc_build_rpc #11 osc_check_rpcs #12 __osc_io_unplug #13 brw_queue_work #14 process_one_work #15 worker_thread #16 kthread #17 ret_from_fork youngest=0s(pid=134199), oldest=0s(pid=134199) ........................ kworker/2:3 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork youngest=786s(pid=124819), oldest=786s(pid=124819) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork youngest=792s(pid=124818), oldest=792s(pid=124818) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 sigsuspend #3 __x64_sys_rt_sigsuspend #4 do_syscall_64 #5 entry_SYSCALL_64_after_hwframe youngest=1797s(pid=96214), oldest=1797s(pid=96214) ........................ anacron 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork youngest=1s(pid=2354), oldest=1s(pid=2354) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork youngest=1s(pid=2353), oldest=1s(pid=2353) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork youngest=3670s(pid=2352), oldest=3670s(pid=2352) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork youngest=0s(pid=2347), oldest=0s(pid=2347) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 do_sigtimedwait #5 __x64_sys_rt_sigtimedwait #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=4856s(pid=1402), oldest=4856s(pid=1402) ........................ (sd-pam) 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_ppoll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1159), oldest=0s(pid=1159) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 __x64_sys_pselect6 #8 do_syscall_64 #9 entry_SYSCALL_64_after_hwframe youngest=377s(pid=1076), oldest=377s(pid=1076) ........................ rsyslogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nfs41_callback_svc #3 kthread #4 ret_from_fork youngest=4575s(pid=976), oldest=4575s(pid=976) ........................ NFSv4 callback 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork youngest=4888s(pid=668), oldest=4888s(pid=668) ........................ jbd2/nbd0-8 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 sk_wait_data #5 tcp_recvmsg #6 inet_recvmsg #7 sock_recvmsg #8 sock_xmit #9 nbd_read_stat #10 recv_work #11 process_one_work #12 worker_thread #13 kthread #14 ret_from_fork youngest=166s(pid=657), oldest=166s(pid=657) ........................ kworker/u9:0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kswapd #3 kthread #4 ret_from_fork youngest=4905s(pid=72), oldest=4905s(pid=72) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthread_worker_fn #3 kthread #4 ret_from_fork youngest=4907s(pid=59), oldest=4907s(pid=59) ........................ watchdogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork youngest=6s(pid=47), oldest=6s(pid=47) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork youngest=4908s(pid=46), oldest=4908s(pid=46) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kcompactd #3 kthread #4 ret_from_fork youngest=4908s(pid=45), oldest=4908s(pid=45) ........................ kcompactd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 oom_reaper #3 kthread #4 ret_from_fork youngest=4907s(pid=43), oldest=4907s(pid=43) ........................ oom_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork youngest=115s(pid=42), oldest=115s(pid=42) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork youngest=790s(pid=39), oldest=790s(pid=39) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 rcu_gp_kthread #4 kthread #5 ret_from_fork youngest=0s(pid=13), oldest=0s(pid=13) ........................ rcu_sched 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork youngest=0s(pid=2), oldest=0s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #0 crash_nmi_callback #1 nmi_handle #2 default_do_nmi #3 do_nmi #4 end_repeat_nmi #-1 native_safe_halt #5 native_safe_halt #6 default_idle #7 arch_cpu_idle #8 default_idle_call #9 do_idle #10 cpu_startup_entry #11 start_secondary #12 secondary_startup_64_no_verify youngest=4908s(pid=0), oldest=4908s(pid=0) ........................ swapper/1 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule_idle #2 do_idle #3 cpu_startup_entry #4 rest_init #5 start_kernel #6 x86_64_start_reservations #7 x86_64_start_kernel #8 secondary_startup_64_no_verify youngest=4908s(pid=0), oldest=4908s(pid=0) ........................ swapper/0 1 times ** Execution took 1.03s (real) 0.25s (CPU), Child processes: 0.79s