***************** crashinfo --stacksummary ***************** /exports/testreports/51304/testresults/sanity2-zfs-centos7_x86_64-centos7_x86_64/oleg319-server-timeout-core (3.10.0-7.9-debug) ------- 105 stacks like that: ---------- #0 __schedule #1 schedule #2 taskq_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=4553), oldest=12142s(pid=3494) ........................ arc_prune 1 times dbu_evict 1 times dp_sync_taskq 9 times dp_zil_clean_ta 12 times metaslab_group_ 6 times spl_delay_taskq 1 times spl_dynamic_tas 1 times spl_kmem_cache 1 times spl_system_task 1 times tx_commit_cb 3 times z_cl_int 3 times z_cl_iss 3 times z_fr_int 3 times z_fr_iss 3 times z_ioctl_int 3 times z_ioctl_iss 3 times z_null_int 3 times z_null_iss 3 times z_prefetch 3 times z_rd_int 3 times z_rd_iss 3 times z_trim_int 3 times z_trim_iss 3 times z_unlinked_drai 3 times z_upgrade 3 times z_vdev_file 1 times z_wr_int 5 times z_wr_int_h 3 times z_wr_iss 5 times z_wr_iss_h 3 times z_zrele 3 times z_zvol 3 times zvol 1 times ------- 70 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=3314), oldest=5070s(pid=4830) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cb00_002 1 times ldlm_cb00_003 1 times ldlm_cb00_004 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn00_003 1 times ldlm_cn00_004 1 times ldlm_cn00_005 1 times ll_mgs_0000 1 times ll_mgs_0001 1 times ll_mgs_0002 1 times ll_ost00_000 1 times ll_ost00_001 1 times ll_ost00_002 1 times ll_ost00_003 1 times ll_ost00_004 1 times ll_ost00_005 1 times ll_ost00_006 1 times ll_ost00_007 1 times ll_ost00_008 1 times ll_ost00_009 1 times ll_ost00_010 1 times ll_ost00_011 1 times ll_ost00_012 1 times ll_ost00_013 1 times ll_ost00_014 1 times ll_ost00_015 1 times ll_ost00_016 1 times ll_ost_create00 3 times ll_ost_io00_000 1 times ll_ost_io00_001 1 times ll_ost_io00_002 1 times ll_ost_io00_003 1 times ll_ost_io00_004 1 times ll_ost_io00_005 1 times ll_ost_io00_006 1 times ll_ost_io00_007 1 times ll_ost_io00_008 1 times ll_ost_io00_009 1 times ll_ost_io00_010 1 times ll_ost_out00_00 3 times ll_ost_seq00_00 2 times mdt00_000 1 times mdt00_001 1 times mdt00_003 1 times mdt00_004 1 times mdt00_005 1 times mdt_fld_0000 1 times mdt_fld_0001 1 times mdt_io00_000 1 times mdt_io00_001 1 times mdt_io00_002 1 times mdt_out00_000 1 times mdt_out00_001 1 times mdt_rdpg00_000 1 times mdt_rdpg00_001 1 times mdt_rdpg00_002 1 times mdt_rdpg00_003 1 times mdt_seqm_0000 1 times mdt_seqm_0001 1 times mdt_seqs_0000 1 times mdt_seqs_0001 1 times ------- 25 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=12183s(pid=722), oldest=12192s(pid=12) ........................ ata_sff 1 times bioset 3 times crypto 1 times deferwq 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kintegrityd 1 times kmpath_rdacd 1 times kpsmoused 1 times kthrotld 1 times kvm-irqfd-clean 1 times lru-add-drain 1 times md 1 times netns 1 times nfsiod 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times watchdogd 1 times writeback 1 times xprtiod 1 times ------- 20 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=18651), oldest=12184s(pid=25) ........................ kworker/0:0 1 times kworker/0:0H 1 times kworker/0:1H 1 times kworker/0:2 1 times kworker/1:0 1 times kworker/1:0H 1 times kworker/1:1 1 times kworker/1:1H 1 times kworker/1:2 1 times kworker/2:0 1 times kworker/2:0H 1 times kworker/2:1H 1 times kworker/2:2 1 times kworker/3:0 1 times kworker/3:0H 1 times kworker/3:1 1 times kworker/3:1H 1 times kworker/3:2 1 times kworker/u8:1 1 times kworker/u8:2 1 times ------- 13 stacks like that: ---------- #0 __schedule #1 schedule #2 cv_wait_common #3 __cv_wait_idle #4 zthr_procedure #5 thread_generic_wrapper #6 kthread #7 ret_from_fork_nospec_begin youngest=346s(pid=4602), oldest=12140s(pid=3500) ........................ arc_evict 1 times z_checkpoint_di 3 times z_indirect_cond 3 times z_livelist_cond 3 times z_livelist_dest 3 times ------- 12 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=7), oldest=3s(pid=28) ........................ ksoftirqd/0 1 times ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_poll #7 system_call_fastpath youngest=1s(pid=831), oldest=12176s(pid=881) ........................ NetworkManager 1 times gdbus 2 times gmain 2 times polkitd 3 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 sys_futex #6 system_call_fastpath youngest=147s(pid=918), oldest=12177s(pid=821) ........................ JS GC Helper 1 times JS Sour~ Thread 1 times gssproxy 5 times rs:main Q:Reg 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_nocb_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=20), oldest=12192s(pid=10) ........................ rcuob/0 1 times rcuob/1 1 times rcuob/2 1 times rcuob/3 1 times rcuos/0 1 times rcuos/1 1 times rcuos/2 1 times rcuos/3 1 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 sys_epoll_wait #6 system_call_fastpath youngest=0s(pid=1), oldest=335s(pid=603) ........................ dbus-daemon 1 times gssproxy 1 times systemd 1 times systemd-journal 1 times systemd-logind 1 times systemd-udevd 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork_nospec_begin youngest=2s(pid=2411), oldest=333s(pid=2407) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_00_02 1 times ptlrpcd_00_03 1 times ptlrpcd_rcv 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=3702), oldest=3s(pid=3800) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ldlm_bl_04 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork_nospec_begin youngest=5061s(pid=2398), oldest=5076s(pid=2396) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 qsd_upd_thread #4 kthread #5 ret_from_fork_nospec_begin youngest=5s(pid=1981), oldest=35s(pid=4972) ........................ lquota_wb_lustr 4 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=2403), oldest=0s(pid=2402) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd00_02 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 mmp_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=4599), oldest=0s(pid=5676) ........................ mmp 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 cv_wait_common #3 __cv_wait_idle #4 txg_quiesce_thread #5 thread_generic_wrapper #6 kthread #7 ret_from_fork_nospec_begin youngest=0s(pid=4597), oldest=3s(pid=1506) ........................ txg_quiesce 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 n_tty_read #4 tty_read #5 vfs_read #6 sys_read #7 system_call_fastpath youngest=12171s(pid=854), oldest=12172s(pid=855) ........................ agetty 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_select #6 core_sys_select #7 sys_select #8 system_call_fastpath youngest=150s(pid=837), oldest=12175s(pid=874) ........................ dhclient 1 times rsyslogd 1 times sshd 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_precreate_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=4845), oldest=4s(pid=4853) ........................ osp-pre-0-0 1 times osp-pre-1-0 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_sync_process_queues #3 llog_process_thread #4 llog_process_or_fork #5 llog_cat_process_cb #6 llog_process_thread #7 llog_process_or_fork #8 llog_cat_process_or_fork #9 llog_cat_process #10 osp_sync_thread #11 kthread #12 ret_from_fork_nospec_begin youngest=157s(pid=4844), oldest=187s(pid=4850) ........................ osp-syn-0-0 1 times osp-syn-1-0 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 mgs_ir_notify #3 kthread #4 ret_from_fork_nospec_begin youngest=335s(pid=4799), oldest=345s(pid=4777) ........................ mgs_lustre_noti 1 times mgs_params_noti 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=2394), oldest=0s(pid=2395) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr00_001 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_inconsistency_verification_main #3 kthread #4 ret_from_fork_nospec_begin youngest=365s(pid=1934), oldest=5064s(pid=5875) ........................ inconsistency_v 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 txg_sync_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork_nospec_begin youngest=0s(pid=5675), oldest=3s(pid=1507) ........................ txg_sync 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork_nospec_begin youngest=12188s(pid=280), oldest=12188s(pid=276) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #-1 poll_idle #0 cpuidle_enter_state #1 cpuidle_idle_call #2 arch_cpu_idle #3 cpu_startup_entry #4 start_secondary #5 start_cpu youngest=12192s(pid=0), oldest=12192s(pid=0) ........................ swapper/1 1 times swapper/2 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule_preempt_disabled #2 __mutex_lock_slowpath #3 mutex_lock #4 vdev_queue_io #5 zio_vdev_io_start #6 zio_nowait #7 vdev_mirror_io_start #8 zio_vdev_io_start #9 zio_execute #10 taskq_thread #11 kthread #12 ret_from_fork_nospec_begin youngest=0s(pid=28112), oldest=0s(pid=28112) ........................ z_wr_iss 1 times ------- 1 stacks like that: ---------- #-1 memset #0 lz4_compress_zfs #1 zio_compress_data #2 zio_write_compress #3 zio_execute #4 taskq_thread #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=28077), oldest=0s(pid=28077) ........................ z_wr_iss 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 distribute_txn_commit_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=335s(pid=4988), oldest=335s(pid=4988) ........................ dist_txn-0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mdt_restriper_main #3 kthread #4 ret_from_fork_nospec_begin youngest=344s(pid=4842), oldest=344s(pid=4842) ........................ mdt_restriper_0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 qmt_reba_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=344s(pid=4841), oldest=344s(pid=4841) ........................ qmt_reba_lustre 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 io_schedule_timeout #4 io_schedule #5 cv_wait_common #6 __cv_wait_io #7 txg_wait_synced_impl #8 txg_wait_synced #9 osd_trans_stop #10 top_trans_stop #11 lod_trans_stop #12 mdd_trans_stop #13 mdd_acl_set #14 mdd_xattr_set #15 mdt_reint_setxattr #16 mdt_reint_rec #17 mdt_reint_internal #18 mdt_reint #19 tgt_request_handle #20 ptlrpc_server_handle_request #21 ptlrpc_main #22 kthread #23 ret_from_fork_nospec_begin youngest=0s(pid=4785), oldest=0s(pid=4785) ........................ mdt00_002 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 io_schedule_timeout #4 __cv_timedwait_common #5 __cv_timedwait_io #6 zio_wait #7 dsl_pool_sync #8 spa_sync #9 txg_sync_thread #10 thread_generic_wrapper #11 kthread #12 ret_from_fork_nospec_begin youngest=0s(pid=4598), oldest=0s(pid=4598) ........................ txg_sync 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=333s(pid=3812), oldest=333s(pid=3812) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ping_evictor_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=3811), oldest=0s(pid=3811) ........................ ll_evictor 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork_nospec_begin youngest=5077s(pid=3801), oldest=5077s(pid=3801) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 l2arc_feed_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork_nospec_begin youngest=0s(pid=3506), oldest=0s(pid=3506) ........................ l2arc_feed 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 dbuf_evict_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=3503), oldest=0s(pid=3503) ........................ dbuf_evict 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 zthr_procedure #7 thread_generic_wrapper #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=3501), oldest=0s(pid=3501) ........................ arc_reap 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=2406), oldest=0s(pid=2406) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=2405), oldest=0s(pid=2405) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork_nospec_begin youngest=5061s(pid=2404), oldest=5061s(pid=2404) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=2400), oldest=0s(pid=2400) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_ppoll #7 system_call_fastpath youngest=0s(pid=916), oldest=0s(pid=916) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule youngest=12178s(pid=782), oldest=12178s(pid=782) ........................ nfsv4.1-svc 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork_nospec_begin youngest=12185s(pid=479), oldest=12185s(pid=479) ........................ jbd2/nbd0-8 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nbd_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=346s(pid=451), oldest=346s(pid=451) ........................ nbd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 sk_wait_data #4 tcp_recvmsg #5 inet_recvmsg #6 sock_recvmsg #7 kernel_recvmsg #8 sock_xmit #9 __nbd_ioctl #10 nbd_ioctl #11 blkdev_ioctl #12 block_ioctl #13 do_vfs_ioctl #14 sys_ioctl #15 system_call_fastpath youngest=346s(pid=449), oldest=346s(pid=449) ........................ nbd-client 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork_nospec_begin youngest=6s(pid=53), oldest=6s(pid=53) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=12190s(pid=52), oldest=12190s(pid=52) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kswapd #3 kthread #4 ret_from_fork_nospec_begin youngest=12190s(pid=51), oldest=12190s(pid=51) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork_nospec_begin youngest=71s(pid=37), oldest=71s(pid=37) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork_nospec_begin youngest=2471s(pid=35), oldest=2471s(pid=35) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 rcu_gp_kthread #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=9), oldest=0s(pid=9) ........................ rcu_sched 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_gp_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=12192s(pid=8), oldest=12192s(pid=8) ........................ rcu_bh 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork_nospec_begin youngest=0s(pid=2), oldest=0s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule_preempt_disabled #2 cpu_startup_entry #3 start_secondary #4 start_cpu youngest=12192s(pid=0), oldest=12192s(pid=0) ........................ swapper/3 1 times ------- 1 stacks like that: ---------- #-1 native_safe_halt #0 default_idle #1 default_enter_idle #2 cpuidle_enter_state #3 cpuidle_idle_call #4 arch_cpu_idle #5 cpu_startup_entry #6 rest_init #7 start_kernel #8 x86_64_start_reservations #9 x86_64_start_kernel #10 start_cpu youngest=12192s(pid=0), oldest=12192s(pid=0) ........................ swapper/0 1 times ** Execution took 1.74s (real) 0.26s (CPU), Child processes: 1.49s