***************** crashinfo --stacksummary ***************** /exports/testreports/69587/testresults/sanityn-ldiskfs-DNE-rocky8.10_x86_64-rocky8.10_x86_64/oleg254-server-timeout-core (4.18.0rh8.10-debug) ------- 69 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork youngest=0s(pid=228212), oldest=2835s(pid=202399) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cb00_002 1 times ldlm_cb00_003 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn00_003 1 times ldlm_cn00_004 1 times ldlm_cn00_005 1 times ll_ost00_000 1 times ll_ost00_001 1 times ll_ost00_002 1 times ll_ost00_003 1 times ll_ost00_004 1 times ll_ost00_005 1 times ll_ost00_006 1 times ll_ost00_007 1 times ll_ost00_008 1 times ll_ost00_009 1 times ll_ost00_010 1 times ll_ost00_011 1 times ll_ost00_012 1 times ll_ost00_013 1 times ll_ost00_014 1 times ll_ost00_015 1 times ll_ost00_016 1 times ll_ost00_017 1 times ll_ost00_018 1 times ll_ost_create00 3 times ll_ost_io00_000 1 times ll_ost_io00_001 1 times ll_ost_io00_002 1 times ll_ost_io00_003 1 times ll_ost_io00_004 1 times ll_ost_out00_00 4 times ll_ost_seq00_00 2 times mdt00_000 1 times mdt00_001 1 times mdt00_002 1 times mdt00_003 1 times mdt00_004 1 times mdt00_005 1 times mdt00_006 1 times mdt_fld_0000 1 times mdt_fld_0001 1 times mdt_io00_000 1 times mdt_io00_001 1 times mdt_io00_002 1 times mdt_io00_003 1 times mdt_io00_004 1 times mdt_out00_000 1 times mdt_out00_001 1 times mdt_out00_002 1 times mdt_out00_003 1 times mdt_rdpg00_000 1 times mdt_rdpg00_001 1 times mdt_rdpg00_002 1 times mdt_rdpg00_003 1 times mdt_seqm_0000 1 times mdt_seqm_0001 1 times mdt_seqs_0000 1 times mdt_seqs_0001 1 times ------- 35 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork youngest=57s(pid=60), oldest=13571s(pid=3) ........................ acpi_thermal_pm 1 times ata_sff 1 times blkcg_punt_bio 1 times crypto 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kdmflush/252:0 1 times kdmflush/252:1 1 times kdmflush/252:2 1 times kdmflush/252:3 1 times kintegrityd 1 times knbd0-recv 1 times kstrp 1 times kthrotld 1 times ldiskfs-misc 4 times md 1 times md_bitmap 1 times mm_percpu_wq 1 times netns 1 times nfsiod 1 times rcu_gp 1 times rcu_par_gp 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times slub_flushwq 1 times tpm_dev_wq 1 times writeback 1 times xprtiod 1 times zswap-shrink 1 times ------- 31 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork youngest=0s(pid=231139), oldest=13560s(pid=658) ........................ kworker/0:0 1 times kworker/0:0H 1 times kworker/0:1 1 times kworker/0:2 1 times kworker/0:2H 1 times kworker/0:3 1 times kworker/0:4 1 times kworker/0:5 1 times kworker/0:6 1 times kworker/0:8 1 times kworker/1:0 1 times kworker/1:1 1 times kworker/1:1H 1 times kworker/1:2 1 times kworker/1:2H 1 times kworker/2:0 1 times kworker/2:1 1 times kworker/2:1H 1 times kworker/2:2 1 times kworker/2:2H 1 times kworker/2:3 1 times kworker/3:0 1 times kworker/3:1H 1 times kworker/3:2H 1 times kworker/3:3 1 times kworker/3:5 1 times kworker/u8:0 1 times kworker/u8:1 1 times kworker/u8:2 1 times kworker/u8:3 1 times kworker/u9:1 1 times ------- 16 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork youngest=0s(pid=32), oldest=13554s(pid=29) ........................ cpuhp/0 1 times cpuhp/1 1 times cpuhp/2 1 times cpuhp/3 1 times ksoftirqd/0 1 times ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 14 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_poll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1020), oldest=13551s(pid=1032) ........................ NetworkManager 1 times gdbus 2 times gmain 3 times irqbalance 1 times polkitd 2 times rngd 1 times rpcbind 1 times sshd 1 times tuned 2 times ------- 10 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 __x64_sys_futex #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1099), oldest=13549s(pid=1055) ........................ JS Helper 4 times gssproxy 5 times rs:main Q:Reg 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 taskq_thread #3 kthread #4 ret_from_fork youngest=13467s(pid=4321), oldest=13469s(pid=4318) ........................ arc_prune 1 times dbu_evict 1 times spl_delay_taskq 1 times spl_dynamic_tas 1 times spl_kmem_cache 1 times spl_system_task 1 times z_vdev_file 1 times zvol_tq-0 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 kern_select #8 __x64_sys_select #9 do_syscall_64 #10 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1369), oldest=13547s(pid=1067) ........................ agetty 3 times dhclient 1 times sshd 2 times tuned 2 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork youngest=0s(pid=3636), oldest=0s(pid=3639) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_00_02 1 times ptlrpcd_00_03 1 times ptlrpcd_rcv 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork youngest=0s(pid=202372), oldest=13560s(pid=670) ........................ jbd2/dm-0-8 1 times jbd2/dm-1-8 1 times jbd2/dm-2-8 1 times jbd2/dm-3-8 1 times jbd2/nbd0-8 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 do_epoll_wait #6 __x64_sys_epoll_wait #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1018), oldest=13549s(pid=1045) ........................ dbus-daemon 1 times gssproxy 1 times systemd 1 times systemd-logind 1 times systemd-udevd 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 qsd_upd_thread #4 kthread #5 ret_from_fork youngest=24s(pid=204720), oldest=57s(pid=203140) ........................ lquota_wb_lustr 4 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork youngest=0s(pid=204293), oldest=0s(pid=202381) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ldlm_bl_04 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork youngest=619s(pid=3625), oldest=13322s(pid=3627) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_precreate_thread #3 kthread #4 ret_from_fork youngest=0s(pid=203128), oldest=3s(pid=203133) ........................ osp-pre-0-1 2 times osp-pre-1-1 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork youngest=2s(pid=3630), oldest=2s(pid=3632) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd00_02 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 do_wait #3 kernel_wait4 #4 __do_sys_wait4 #5 __x64_sys_wait4 #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=0s(pid=242306), oldest=0s(pid=242321) ........................ bash 2 times ------- 2 stacks like that: ---------- #-1 smp_call_function_many_cond #0 on_each_cpu #1 flush_tlb_kernel_range #2 free_unmap_vmap_area #3 remove_vm_area #4 __vunmap #5 vfree #6 free_work #7 process_one_work #8 worker_thread #9 kthread #10 ret_from_fork youngest=0s(pid=241465), oldest=0s(pid=242140) ........................ kworker/0:7 1 times kworker/1:3 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_id_repair_thread_main #3 kthread #4 ret_from_fork youngest=2788s(pid=204709), oldest=2804s(pid=203905) ........................ ofd_id_repair 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_inconsistency_verification_main #3 kthread #4 ret_from_fork youngest=2788s(pid=204708), oldest=2804s(pid=203904) ........................ inconsistency_v 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_sync_process_queues #3 llog_process_thread #4 llog_process_or_fork #5 llog_cat_process_cb #6 llog_process_thread #7 llog_process_or_fork #8 llog_cat_process_or_fork #9 llog_cat_process #10 osp_sync_thread #11 kthread #12 ret_from_fork youngest=0s(pid=203130), oldest=6s(pid=203132) ........................ osp-syn-0-1 1 times osp-syn-1-1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 zthr_procedure #7 thread_generic_wrapper #8 kthread #9 ret_from_fork youngest=0s(pid=4325), oldest=0s(pid=4326) ........................ arc_evict 1 times arc_reap 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork youngest=11s(pid=3624), oldest=12s(pid=3623) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr00_001 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 handle_mm_fault #6 do_epoll_wait #7 __x64_sys_epoll_wait #8 do_syscall_64 #9 entry_SYSCALL_64_after_hwframe youngest=0s(pid=790), oldest=0s(pid=242139) ........................ systemd 1 times systemd-journal 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork youngest=13565s(pid=471), oldest=13565s(pid=474) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_tasks_kthread #3 kthread #4 ret_from_fork youngest=13571s(pid=11), oldest=13571s(pid=10) ........................ rcu_tasks_rude_ 1 times rcu_tasks_trace 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule_idle #2 do_idle #3 cpu_startup_entry #4 start_secondary #5 secondary_startup_64_no_verify youngest=13571s(pid=0), oldest=13571s(pid=0) ........................ swapper/1 1 times swapper/2 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 wb_wait_for_completion #3 sync_inodes_sb #4 __sync_filesystem #5 sync_filesystem #6 generic_shutdown_super #7 kill_block_super #8 deactivate_locked_super #9 deactivate_super #10 cleanup_mnt #11 mntput_no_expire #12 mntput #13 osd_umount #14 osd_device_fini #15 obd_precleanup #16 class_cleanup #17 class_process_config #18 class_manual_cleanup #19 osd_obd_disconnect #20 lustre_put_lsi #21 server_put_super #22 generic_shutdown_super #23 kill_anon_super #24 lustre_kill_super #25 deactivate_locked_super #26 deactivate_super #27 cleanup_mnt #28 __cleanup_mnt #29 task_work_run #30 exit_to_usermode_loop #31 do_syscall_64 #32 entry_SYSCALL_64_after_hwframe youngest=0s(pid=242322), oldest=0s(pid=242322) ........................ umount 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 do_sigtimedwait #5 __x64_sys_rt_sigtimedwait #6 do_syscall_64 #7 entry_SYSCALL_64_after_hwframe youngest=7s(pid=242144), oldest=7s(pid=242144) ........................ (sd-pam) 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 distribute_txn_commit_thread #4 kthread #5 ret_from_fork youngest=6s(pid=203142), oldest=6s(pid=203142) ........................ dist_txn-1 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_send_update_thread #3 kthread #4 ret_from_fork youngest=6s(pid=203127), oldest=6s(pid=203127) ........................ osp_up0-1 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mdt_restriper_main #3 kthread #4 ret_from_fork youngest=2821s(pid=203125), oldest=2821s(pid=203125) ........................ mdt_restriper_0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork youngest=2548s(pid=202389), oldest=2548s(pid=202389) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ping_evictor_main #3 kthread #4 ret_from_fork youngest=0s(pid=202388), oldest=0s(pid=202388) ........................ ll_evictor 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork youngest=2835s(pid=202382), oldest=2835s(pid=202382) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 l2arc_feed_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork youngest=0s(pid=4331), oldest=0s(pid=4331) ........................ l2arc_feed 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 dbuf_evict_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork youngest=0s(pid=4328), oldest=0s(pid=4328) ........................ dbuf_evict 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork youngest=0s(pid=3635), oldest=0s(pid=3635) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork youngest=0s(pid=3634), oldest=0s(pid=3634) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork youngest=619s(pid=3633), oldest=619s(pid=3633) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork youngest=0s(pid=3629), oldest=0s(pid=3629) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_sys_poll #6 __x64_sys_ppoll #7 do_syscall_64 #8 entry_SYSCALL_64_after_hwframe youngest=0s(pid=1097), oldest=0s(pid=1097) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout.constprop.11 #5 do_select #6 core_sys_select #7 __x64_sys_pselect6 #8 do_syscall_64 #9 entry_SYSCALL_64_after_hwframe youngest=303s(pid=1083), oldest=303s(pid=1083) ........................ rsyslogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 do_nanosleep #3 hrtimer_nanosleep #4 __x64_sys_nanosleep #5 do_syscall_64 #6 entry_SYSCALL_64_after_hwframe youngest=11s(pid=1059), oldest=11s(pid=1059) ........................ crond 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nfs41_callback_svc #3 kthread #4 ret_from_fork youngest=13553s(pid=983), oldest=13553s(pid=983) ........................ NFSv4 callback 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 sk_wait_data #5 tcp_recvmsg #6 inet_recvmsg #7 sock_recvmsg #8 sock_xmit #9 nbd_read_stat #10 recv_work #11 process_one_work #12 worker_thread #13 kthread #14 ret_from_fork youngest=10823s(pid=655), oldest=10823s(pid=655) ........................ kworker/u9:0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kswapd #3 kthread #4 ret_from_fork youngest=13568s(pid=71), oldest=13568s(pid=71) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthread_worker_fn #3 kthread #4 ret_from_fork youngest=13570s(pid=58), oldest=13570s(pid=58) ........................ watchdogd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork youngest=0s(pid=47), oldest=0s(pid=47) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork youngest=13571s(pid=46), oldest=13571s(pid=46) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kcompactd #3 kthread #4 ret_from_fork youngest=13571s(pid=45), oldest=13571s(pid=45) ........................ kcompactd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 oom_reaper #3 kthread #4 ret_from_fork youngest=13571s(pid=43), oldest=13571s(pid=43) ........................ oom_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork youngest=53s(pid=42), oldest=53s(pid=42) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork youngest=13340s(pid=39), oldest=13340s(pid=39) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #-1 do_raw_spin_unlock #0 _raw_spin_unlock_irqrestore #1 debug_object_assert_init #2 __try_to_del_timer_sync #3 __timer_delete_sync #4 schedule_timeout #5 rcu_gp_kthread #6 kthread #7 ret_from_fork youngest=0s(pid=13), oldest=0s(pid=13) ........................ rcu_sched 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork youngest=0s(pid=2), oldest=0s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #-1 _raw_spin_unlock_irqrestore #0 percpu_counter_add_batch #1 test_clear_page_writeback #2 end_page_writeback #3 end_buffer_async_write #4 end_bio_bh_io_sync #5 bio_endio #6 dm_io_complete #7 dm_io_dec_pending #8 clone_endio #9 bio_endio #10 blk_update_request #11 blk_mq_end_request #12 virtblk_request_done #13 blk_mq_complete_request #14 virtblk_done #15 vring_interrupt #16 __handle_irq_event_percpu #17 handle_irq_event_percpu #18 handle_irq_event #19 handle_edge_irq #20 handle_irq #21 do_IRQ #22 common_interrupt #23 __softirqentry_text_start #24 irq_exit_rcu #25 irq_exit #26 smp_apic_timer_interrupt #27 apic_timer_interrupt #28 apic_timer_interrupt #-1 native_safe_halt #29 default_idle #30 arch_cpu_idle #31 default_idle_call #32 do_idle #33 cpu_startup_entry #34 start_secondary #35 secondary_startup_64_no_verify youngest=13571s(pid=0), oldest=13571s(pid=0) ........................ swapper/3 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule_idle #2 do_idle #3 cpu_startup_entry #4 rest_init #5 start_kernel #6 x86_64_start_reservations #7 x86_64_start_kernel #8 secondary_startup_64_no_verify youngest=13571s(pid=0), oldest=13571s(pid=0) ........................ swapper/0 1 times ** Execution took 1.46s (real) 0.35s (CPU), Child processes: 1.12s