***************** crashinfo --stacksummary ***************** /exports/testreports/58918/testresults/recovery-small-zfs-rocky8.10_x86_64-rocky8.10_x86_64-retry1/oleg214-server-timeout-core (4.18.0rh8.10-debug) ------- 113 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork youngest=1084s(pid=108304), oldest=9675s(pid=14928) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn00_003 1 times ldlm_cn00_004 1 times ldlm_cn00_005 1 times ldlm_cn00_006 1 times ldlm_cn00_007 1 times ll_ost00_000 1 times ll_ost00_001 1 times ll_ost00_002 1 times ll_ost00_003 1 times ll_ost00_004 1 times ll_ost00_005 1 times ll_ost00_006 1 times ll_ost00_007 1 times ll_ost00_008 1 times ll_ost00_009 1 times ll_ost00_010 1 times ll_ost00_011 1 times ll_ost00_012 1 times ll_ost00_013 1 times ll_ost00_014 1 times ll_ost00_015 1 times ll_ost00_016 1 times ll_ost00_017 1 times ll_ost00_018 1 times ll_ost00_019 1 times ll_ost00_020 1 times ll_ost00_021 1 times ll_ost00_022 1 times ll_ost00_023 1 times ll_ost00_024 1 times ll_ost00_025 1 times ll_ost00_026 1 times ll_ost00_027 1 times ll_ost00_028 1 times ll_ost00_029 1 times ll_ost00_030 1 times ll_ost00_031 1 times ll_ost00_032 1 times ll_ost00_033 1 times ll_ost00_034 1 times ll_ost00_035 1 times ll_ost00_036 1 times ll_ost00_037 1 times ll_ost00_038 1 times ll_ost00_039 1 times ll_ost00_040 1 times ll_ost00_041 1 times ll_ost00_042 1 times ll_ost00_043 1 times ll_ost00_044 1 times ll_ost00_045 1 times ll_ost00_046 1 times ll_ost00_047 1 times ll_ost00_048 1 times ll_ost00_049 1 times ll_ost00_050 1 times ll_ost00_051 1 times ll_ost00_052 1 times ll_ost00_053 1 times ll_ost00_054 1 times ll_ost00_055 1 times ll_ost00_056 1 times ll_ost00_057 1 times ll_ost00_058 1 times ll_ost00_059 1 times ll_ost00_060 1 times ll_ost00_061 1 times ll_ost00_062 1 times ll_ost00_063 1 times ll_ost00_064 1 times ll_ost00_065 1 times ll_ost00_066 1 times ll_ost00_067 1 times ll_ost00_068 1 times ll_ost00_069 1 times ll_ost00_070 1 times ll_ost00_071 1 times ll_ost00_072 1 times ll_ost00_073 1 times ll_ost00_074 1 times ll_ost00_075 1 times ll_ost00_076 1 times ll_ost00_077 1 times ll_ost00_078 1 times ll_ost00_079 1 times ll_ost00_080 1 times ll_ost00_081 1 times ll_ost00_082 1 times ll_ost_create00 3 times ll_ost_io00_000 1 times ll_ost_io00_001 1 times ll_ost_io00_002 1 times ll_ost_io00_003 1 times ll_ost_io00_004 1 times ll_ost_io00_005 1 times ll_ost_io00_006 1 times ll_ost_io00_007 1 times ll_ost_io00_008 1 times ll_ost_io00_009 1 times ll_ost_io00_010 1 times ll_ost_io00_011 1 times ll_ost_out00_00 2 times ll_ost_seq00_00 3 times ------- 106 stacks like that: ---------- #0 __schedule #1 schedule #2 taskq_thread #3 kthread #4 ret_from_fork youngest=0s(pid=122756), oldest=9974s(pid=2591) ........................ arc_prune 1 times dbu_evict 1 times dp_sync_taskq 3 times dp_zil_clean_ta 12 times spl_delay_taskq 1 times spl_dynamic_tas 1 times spl_kmem_cache 1 times spl_system_task 1 times tx_commit_cb 2 times z_cl_int 3 times z_cl_iss 3 times z_flush_int 3 times z_flush_iss 3 times z_fr_int 3 times z_fr_iss 3 times z_metaslab 6 times z_null_int 3 times z_null_iss 3 times z_prefetch 3 times z_rd_int 9 times z_rd_iss 3 times z_trim_int 3 times z_trim_iss 3 times z_unlinked_drai 3 times z_upgrade 3 times z_vdev_file 1 times z_wr_int 9 times z_wr_iss 9 times z_zrele 3 times z_zvol 3 times zvol_tq-0 1 times ------- 26 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork youngest=10025s(pid=675), oldest=10038s(pid=3) ........................ acpi_thermal_pm 1 times ata_sff 1 times blkcg_punt_bio 1 times crypto 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kintegrityd 1 times kstrp 1 times kthrotld 1 times md 1 times md_bitmap 1 times mm_percpu_wq 1 times netns 1 times nfsiod 1 times rcu_gp 1 times rcu_par_gp 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times slub_flushwq 1 times tpm_dev_wq 1 times writeback 1 times xprtiod 1 times zswap-shrink 1 times ------- 23 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork youngest=0s(pid=129068), oldest=10026s(pid=666) ........................ kworker/0:0 1 times kworker/0:0H 1 times kworker/0:1 1 times kworker/0:2 1 times kworker/0:2H 1 times kworker/1:0 1 times kworker/1:1 1 times kworker/1:1H 1 times kworker/1:2H 1 times kworker/2:0 1 times kworker/2:0H 1 times kworker/2:2H 1 times kworker/2:6 1 times kworker/3:0 1 times kworker/3:1 1 times kworker/3:1H 1 times kworker/3:2 1 times kworker/3:2H 1 times kworker/u8:0 1 times kworker/u8:2 1 times kworker/u8:3 1 times kworker/u9:0 1 times kworker/u9:1 1 times ------- 16 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork youngest=2s(pid=30), oldest=10020s(pid=17) ........................ cpuhp/0 1 times cpuhp/1 1 times cpuhp/2 1 times cpuhp/3 1 times ksoftirqd/0 1 times ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 15 stacks like that: ---------- #0 __schedule #1 schedule #2 cv_wait_common #3 __cv_wait_idle #4 zthr_procedure #5 thread_generic_wrapper #6 kthread #7 ret_from_fork youngest=1686s(pid=128598), oldest=9689s(pid=14706) ........................ raidz_expand 3 times z_checkpoint_di 3 times z_indirect_cond 3 times z_livelist_cond 3 times z_livelist_dest 3 times ------- 13 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_poll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1019), oldest=10013s(pid=1027) ........................ NetworkManager 1 times gdbus 2 times gmain 3 times irqbalance 1 times polkitd 2 times rngd 1 times rpcbind 1 times tuned 2 times ------- 10 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 __x64_sys_futex #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=1071s(pid=1183), oldest=10012s(pid=1056) ........................ JS Helper 4 times gssproxy 5 times rs:main Q:Reg 1 times ------- 7 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 kern_select #8 __x64_sys_select #9 do_syscall_64 #10 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1048), oldest=10010s(pid=1072) ........................ agetty 3 times dhclient 1 times sshd 1 times tuned 2 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 do_epoll_wait #6 __x64_sys_epoll_wait #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=34s(pid=1), oldest=10012s(pid=1049) ........................ dbus-daemon 1 times gssproxy 1 times systemd 1 times systemd-journal 1 times systemd-logind 1 times systemd-udevd 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork youngest=0s(pid=33267), oldest=4s(pid=38503) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ldlm_bl_04 1 times ldlm_bl_05 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork youngest=0s(pid=6199), oldest=1605s(pid=6201) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_00_02 1 times ptlrpcd_00_03 1 times ptlrpcd_rcv 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork youngest=9670s(pid=6191), oldest=9719s(pid=6188) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 mmp_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork youngest=0s(pid=122801), oldest=0s(pid=14704) ........................ mmp 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 txg_sync_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork youngest=0s(pid=14703), oldest=1s(pid=128595) ........................ txg_sync 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 cv_wait_common #3 __cv_wait_idle #4 txg_quiesce_thread #5 thread_generic_wrapper #6 kthread #7 ret_from_fork youngest=0s(pid=14702), oldest=1s(pid=122799) ........................ txg_quiesce 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork youngest=7s(pid=6194), oldest=14s(pid=6195) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd00_02 1 times ------- 3 stacks like that: ---------- #-1 native_safe_halt #0 default_idle #1 arch_cpu_idle #2 default_idle_call #3 do_idle #4 cpu_startup_entry #5 start_secondary #6 secondary_startup_64_no_verify youngest=10038s(pid=0), oldest=10038s(pid=0) ........................ swapper/1 1 times swapper/2 1 times swapper/3 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 qsd_upd_thread #4 kthread #5 ret_from_fork youngest=2s(pid=122930), oldest=39s(pid=14971) ........................ lquota_wb_lustr 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_id_repair_thread_main #3 kthread #4 ret_from_fork youngest=2093s(pid=122923), oldest=9684s(pid=14958) ........................ ofd_id_repair 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_inconsistency_verification_main #3 kthread #4 ret_from_fork youngest=2093s(pid=122922), oldest=9685s(pid=14956) ........................ inconsistency_v 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork youngest=1714s(pid=6187), oldest=1714s(pid=6186) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr00_001 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 zthr_procedure #7 thread_generic_wrapper #8 kthread #9 ret_from_fork youngest=0s(pid=2599), oldest=0s(pid=2598) ........................ arc_evict 1 times arc_reap 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork youngest=10032s(pid=461), oldest=10032s(pid=478) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_tasks_kthread #3 kthread #4 ret_from_fork youngest=10038s(pid=11), oldest=10038s(pid=10) ........................ rcu_tasks_rude_ 1 times rcu_tasks_trace 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork youngest=1963s(pid=12907), oldest=1963s(pid=12907) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ping_evictor_main #3 kthread #4 ret_from_fork youngest=1090s(pid=12906), oldest=1090s(pid=12906) ........................ ll_evictor 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork youngest=3737s(pid=12900), oldest=3737s(pid=12900) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork youngest=0s(pid=6198), oldest=0s(pid=6198) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork youngest=0s(pid=6197), oldest=0s(pid=6197) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork youngest=9670s(pid=6196), oldest=9670s(pid=6196) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork youngest=0s(pid=6192), oldest=0s(pid=6192) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 l2arc_feed_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork youngest=0s(pid=2604), oldest=0s(pid=2604) ........................ l2arc_feed 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 dbuf_evict_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork youngest=0s(pid=2601), oldest=0s(pid=2601) ........................ dbuf_evict 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_ppoll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1180), oldest=0s(pid=1180) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 __x64_sys_pselect6 #8 do_syscall_64 #9 entry_SYSCALL_64_after_hwframe youngest=144s(pid=1088), oldest=144s(pid=1088) ........................ rsyslogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 do_nanosleep #3 hrtimer_nanosleep #4 __x64_sys_nanosleep #5 do_syscall_64 #6 entry_SYSCALL_64_after_hwframe youngest=58s(pid=1063), oldest=58s(pid=1063) ........................ crond 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nfs41_callback_svc #3 kthread #4 ret_from_fork youngest=10016s(pid=983), oldest=10016s(pid=983) ........................ NFSv4 callback 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork youngest=10025s(pid=674), oldest=10025s(pid=674) ........................ jbd2/nbd0-8 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 sk_wait_data #5 tcp_recvmsg #6 inet_recvmsg #7 sock_recvmsg #8 sock_xmit #9 nbd_read_stat #10 recv_work #11 process_one_work #12 rescuer_thread #13 kthread #14 ret_from_fork youngest=1686s(pid=661), oldest=1686s(pid=661) ........................ knbd0-recv 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kswapd #3 kthread #4 ret_from_fork youngest=10035s(pid=71), oldest=10035s(pid=71) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthread_worker_fn #3 kthread #4 ret_from_fork youngest=10037s(pid=59), oldest=10037s(pid=59) ........................ watchdogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork youngest=9s(pid=48), oldest=9s(pid=48) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork youngest=10038s(pid=47), oldest=10038s(pid=47) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kcompactd #3 kthread #4 ret_from_fork youngest=10038s(pid=46), oldest=10038s(pid=46) ........................ kcompactd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 oom_reaper #3 kthread #4 ret_from_fork youngest=10038s(pid=44), oldest=10038s(pid=44) ........................ oom_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork youngest=84s(pid=43), oldest=84s(pid=43) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork youngest=3455s(pid=39), oldest=3455s(pid=39) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_gp_kthread #3 kthread #4 ret_from_fork youngest=3s(pid=13), oldest=3s(pid=13) ........................ rcu_sched 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork youngest=6s(pid=2), oldest=6s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #-1 native_safe_halt #0 default_idle #1 arch_cpu_idle #2 default_idle_call #3 do_idle #4 cpu_startup_entry #5 rest_init #6 start_kernel #7 x86_64_start_reservations #8 x86_64_start_kernel #9 secondary_startup_64_no_verify youngest=10038s(pid=0), oldest=10038s(pid=0) ........................ swapper/0 1 times ** Execution took 2.35s (real) 0.68s (CPU), Child processes: 1.70s