***************** crashinfo --stacksummary ***************** /exports/testreports/47845/testresults/sanity-benchmark-special2-ldiskfs-DNE-centos7_x86_64-centos7_x86_64/oleg151-server-timeout-core (3.10.0-7.9-debug) ------- 113 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=9572), oldest=319s(pid=7238) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cb00_002 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn00_003 1 times ldlm_cn00_004 1 times ldlm_cn00_005 1 times ldlm_cn00_006 1 times ldlm_cn00_007 1 times ldlm_cn00_008 1 times ldlm_cn00_009 1 times ldlm_cn00_010 1 times ll_mgs_0000 1 times ll_mgs_0001 1 times ll_mgs_0002 1 times ll_ost00_000 1 times ll_ost00_001 1 times ll_ost00_002 1 times ll_ost00_003 1 times ll_ost00_004 1 times ll_ost00_005 1 times ll_ost00_006 1 times ll_ost00_007 1 times ll_ost00_008 1 times ll_ost00_009 1 times ll_ost00_010 1 times ll_ost00_011 1 times ll_ost00_012 1 times ll_ost00_013 1 times ll_ost00_014 1 times ll_ost00_015 1 times ll_ost00_016 1 times ll_ost00_017 1 times ll_ost00_018 1 times ll_ost00_019 1 times ll_ost00_020 1 times ll_ost00_021 1 times ll_ost00_022 1 times ll_ost00_023 1 times ll_ost00_024 1 times ll_ost00_025 1 times ll_ost00_026 1 times ll_ost00_027 1 times ll_ost00_028 1 times ll_ost00_029 1 times ll_ost00_030 1 times ll_ost00_031 1 times ll_ost00_032 1 times ll_ost00_033 1 times ll_ost_create00 3 times ll_ost_io00_000 1 times ll_ost_io00_001 1 times ll_ost_io00_002 1 times ll_ost_io00_003 1 times ll_ost_io00_005 1 times ll_ost_io00_007 1 times ll_ost_io00_009 1 times ll_ost_io00_010 1 times ll_ost_io00_012 1 times ll_ost_io00_014 1 times ll_ost_io00_016 1 times ll_ost_io00_017 1 times ll_ost_io00_018 1 times ll_ost_io00_019 1 times ll_ost_out00_00 2 times ll_ost_seq00_00 2 times mdt00_000 1 times mdt00_001 1 times mdt00_002 1 times mdt00_003 1 times mdt00_004 1 times mdt00_005 1 times mdt00_006 1 times mdt00_007 1 times mdt00_008 1 times mdt00_009 1 times mdt00_010 1 times mdt00_011 1 times mdt00_012 1 times mdt00_013 1 times mdt00_014 1 times mdt_fld_0000 1 times mdt_fld_0001 1 times mdt_fld_0002 1 times mdt_io00_000 1 times mdt_io00_001 1 times mdt_io00_002 1 times mdt_io00_003 1 times mdt_io00_004 1 times mdt_out00_000 1 times mdt_out00_001 1 times mdt_out00_002 1 times mdt_rdpg00_000 1 times mdt_rdpg00_001 1 times mdt_rdpg00_002 1 times mdt_rdpg00_003 1 times mdt_rdpg00_004 1 times mdt_rdpg00_005 1 times mdt_rdpg00_006 1 times mdt_rdpg00_007 1 times mdt_rdpg00_008 1 times mdt_rdpg00_009 1 times mdt_rdpg00_010 1 times mdt_seqm_0000 1 times mdt_seqm_0001 1 times mdt_seqs_0000 1 times mdt_seqs_0001 1 times ------- 37 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=308s(pid=10846), oldest=411s(pid=12) ........................ ata_sff 1 times bioset 7 times crypto 1 times deferwq 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kdmflush 4 times kintegrityd 1 times kmpath_rdacd 1 times kpsmoused 1 times kthrotld 1 times kvm-irqfd-clean 1 times ldiskfs-misc 4 times lru-add-drain 1 times md 1 times netns 1 times nfsiod 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times watchdogd 1 times writeback 1 times xprtiod 1 times ------- 21 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=16370), oldest=401s(pid=25) ........................ kworker/0:0 1 times kworker/0:0H 1 times kworker/0:1 1 times kworker/0:1H 1 times kworker/0:2 1 times kworker/1:0H 1 times kworker/1:1 1 times kworker/1:1H 1 times kworker/1:2 1 times kworker/2:0 1 times kworker/2:0H 1 times kworker/2:1 1 times kworker/2:1H 1 times kworker/2:2 1 times kworker/3:0 1 times kworker/3:0H 1 times kworker/3:1 1 times kworker/3:1H 1 times kworker/3:2 1 times kworker/u8:0 1 times kworker/u8:3 1 times ------- 11 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=7), oldest=3s(pid=23) ........................ ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 taskq_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=350s(pid=4529), oldest=352s(pid=4514) ........................ arc_prune 1 times dbu_evict 1 times spl_delay_taskq 1 times spl_dynamic_tas 1 times spl_kmem_cache 1 times spl_system_task 1 times z_vdev_file 1 times zvol 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 sys_futex #6 system_call_fastpath youngest=77s(pid=926), oldest=384s(pid=824) ........................ JS GC Helper 1 times JS Sour~ Thread 1 times gssproxy 5 times rs:main Q:Reg 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_poll #7 system_call_fastpath youngest=3s(pid=832), oldest=383s(pid=881) ........................ NetworkManager 1 times gdbus 2 times gmain 2 times polkitd 3 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_nocb_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=11), oldest=411s(pid=10) ........................ rcuob/0 1 times rcuob/1 1 times rcuob/2 1 times rcuob/3 1 times rcuos/0 1 times rcuos/1 1 times rcuos/2 1 times rcuos/3 1 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_precreate_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=10102), oldest=3s(pid=8381) ........................ osp-pre-0-0 1 times osp-pre-0-1 2 times osp-pre-1-0 2 times osp-pre-1-1 1 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 qsd_upd_thread #4 kthread #5 ret_from_fork_nospec_begin youngest=8s(pid=10858), oldest=25s(pid=7255) ........................ lquota_wb_lustr 6 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 sys_epoll_wait #6 system_call_fastpath youngest=0s(pid=1), oldest=248s(pid=598) ........................ dbus-daemon 1 times gssproxy 1 times systemd 1 times systemd-journal 1 times systemd-logind 1 times systemd-udevd 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=3770), oldest=302s(pid=3766) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_00_02 1 times ptlrpcd_00_03 1 times ptlrpcd_rcv 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork_nospec_begin youngest=2s(pid=10845), oldest=396s(pid=484) ........................ jbd2/dm-0-8 1 times jbd2/dm-1-8 1 times jbd2/dm-2-8 1 times jbd2/dm-3-8 1 times jbd2/nbd0-8 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_sync_process_queues #3 llog_process_thread #4 llog_process_or_fork #5 llog_cat_process_cb #6 llog_process_thread #7 llog_process_or_fork #8 llog_cat_process_or_fork #9 llog_cat_process #10 osp_sync_thread #11 kthread #12 ret_from_fork_nospec_begin youngest=65s(pid=10101), oldest=77s(pid=10109) ........................ osp-syn-0-0 1 times osp-syn-0-1 1 times osp-syn-1-0 1 times osp-syn-1-1 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork_nospec_begin youngest=305s(pid=3756), oldest=325s(pid=3755) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=7216), oldest=2s(pid=8376) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=3762), oldest=0s(pid=3761) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd00_02 1 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 n_tty_read #4 tty_read #5 vfs_read #6 sys_read #7 system_call_fastpath youngest=379s(pid=854), oldest=380s(pid=852) ........................ agetty 3 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_select #6 core_sys_select #7 sys_select #8 system_call_fastpath youngest=77s(pid=836), oldest=383s(pid=903) ........................ dhclient 1 times rsyslogd 1 times sshd 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 target_bulk_io #4 tgt_brw_write #5 tgt_request_handle #6 ptlrpc_server_handle_request #7 ptlrpc_main #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=15940), oldest=0s(pid=15917) ........................ ll_ost_io00_004 1 times ll_ost_io00_008 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ofd_inconsistency_verification_main #3 kthread #4 ret_from_fork_nospec_begin youngest=308s(pid=10853), oldest=313s(pid=9585) ........................ inconsistency_v 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 osp_send_update_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=76s(pid=8380), oldest=311s(pid=10106) ........................ osp_up0-1 1 times osp_up1-0 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 distribute_txn_commit_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=76s(pid=8387), oldest=325s(pid=7256) ........................ dist_txn-0 1 times dist_txn-1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 mdt_restriper_main #3 kthread #4 ret_from_fork_nospec_begin youngest=319s(pid=8379), oldest=325s(pid=7250) ........................ mdt_restriper_0 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 mgs_ir_notify #3 kthread #4 ret_from_fork_nospec_begin youngest=308s(pid=7242), oldest=325s(pid=7220) ........................ mgs_lustre_noti 1 times mgs_params_noti 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork_nospec_begin youngest=71s(pid=3753), oldest=72s(pid=3754) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr00_001 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork_nospec_begin youngest=403s(pid=294), oldest=403s(pid=291) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_gp_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=9), oldest=411s(pid=8) ........................ rcu_bh 1 times rcu_sched 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule_preempt_disabled #2 cpu_startup_entry #3 start_secondary #4 start_cpu youngest=413s(pid=0), oldest=413s(pid=0) ........................ swapper/1 1 times swapper/2 1 times ------- 1 stacks like that: ---------- #-1 __pv_queued_spin_lock_slowpath #0 do_raw_spin_lock #1 _raw_spin_lock_irq #2 __add_to_page_cache_locked #3 add_to_page_cache_lru #4 find_or_create_page #5 osd_bufs_get #6 ofd_preprw_write #7 ofd_preprw #8 tgt_brw_write #9 tgt_request_handle #10 ptlrpc_server_handle_request #11 ptlrpc_main #12 kthread #13 ret_from_fork_nospec_begin youngest=0s(pid=15967), oldest=0s(pid=15967) ........................ ll_ost_io00_015 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 osd_trans_stop #3 ofd_trans_stop #4 ofd_commitrw_write #5 ofd_commitrw #6 tgt_brw_write #7 tgt_request_handle #8 ptlrpc_server_handle_request #9 ptlrpc_main #10 kthread #11 ret_from_fork_nospec_begin youngest=0s(pid=15965), oldest=0s(pid=15965) ........................ ll_ost_io00_013 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 __cond_resched #2 _cond_resched #3 flush_work #4 __cancel_work_timer #5 cancel_delayed_work_sync #6 ptlrpc_wait_event #7 ptlrpc_main #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=15963), oldest=0s(pid=15963) ........................ ll_ost_io00_011 1 times ------- 1 stacks like that: ---------- #-1 crc_128 #0 crc32c_pcl_intel_update #1 crypto_shash_update #2 shash_ahash_update #3 shash_async_update #4 cfs_crypto_hash_update_page #5 tgt_checksum_niobuf_rw #6 tgt_brw_write #7 tgt_request_handle #8 ptlrpc_server_handle_request #9 ptlrpc_main #10 kthread #11 ret_from_fork_nospec_begin youngest=0s(pid=15927), oldest=0s(pid=15927) ........................ ll_ost_io00_006 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 qmt_reba_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=325s(pid=7249), oldest=325s(pid=7249) ........................ qmt_reba_lustre 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=251s(pid=7225), oldest=251s(pid=7225) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ping_evictor_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=7224), oldest=0s(pid=7224) ........................ ll_evictor 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork_nospec_begin youngest=326s(pid=7218), oldest=326s(pid=7218) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 __cv_timedwait_common #4 __cv_timedwait_idle #5 l2arc_feed_thread #6 thread_generic_wrapper #7 kthread #8 ret_from_fork_nospec_begin youngest=0s(pid=4530), oldest=0s(pid=4530) ........................ l2arc_feed 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 dbuf_evict_thread #7 thread_generic_wrapper #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=4528), oldest=0s(pid=4528) ........................ dbuf_evict 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 cv_timedwait_hires_common #5 cv_timedwait_idle_hires #6 zthr_procedure #7 thread_generic_wrapper #8 kthread #9 ret_from_fork_nospec_begin youngest=0s(pid=4526), oldest=0s(pid=4526) ........................ arc_reap 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 cv_wait_common #3 __cv_wait_idle #4 zthr_procedure #5 thread_generic_wrapper #6 kthread #7 ret_from_fork_nospec_begin youngest=350s(pid=4525), oldest=350s(pid=4525) ........................ arc_evict 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=3765), oldest=0s(pid=3765) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=3764), oldest=0s(pid=3764) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork_nospec_begin youngest=305s(pid=3763), oldest=305s(pid=3763) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=3759), oldest=0s(pid=3759) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_ppoll #7 system_call_fastpath youngest=0s(pid=923), oldest=0s(pid=923) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule youngest=385s(pid=780), oldest=385s(pid=780) ........................ nfsv4.1-svc 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nbd_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=288s(pid=455), oldest=288s(pid=455) ........................ nbd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 sk_wait_data #4 tcp_recvmsg #5 inet_recvmsg #6 sock_recvmsg #7 kernel_recvmsg #8 sock_xmit #9 __nbd_ioctl #10 nbd_ioctl #11 blkdev_ioctl #12 block_ioctl #13 do_vfs_ioctl #14 sys_ioctl #15 system_call_fastpath youngest=288s(pid=453), oldest=288s(pid=453) ........................ nbd-client 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork_nospec_begin youngest=4s(pid=55), oldest=4s(pid=55) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=407s(pid=54), oldest=407s(pid=54) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #-1 __pv_queued_spin_lock_slowpath #0 do_raw_spin_lock #1 _raw_spin_lock_irq #2 __remove_mapping #3 shrink_page_list #4 shrink_inactive_list #5 shrink_lruvec #6 shrink_zone #7 balance_pgdat #8 kswapd #9 kthread #10 ret_from_fork_nospec_begin youngest=0s(pid=53), oldest=0s(pid=53) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork_nospec_begin youngest=51s(pid=37), oldest=51s(pid=37) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork_nospec_begin youngest=309s(pid=35), oldest=309s(pid=35) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 __cond_resched #2 _cond_resched #3 run_ksoftirqd #4 smpboot_thread_fn #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=6), oldest=0s(pid=6) ........................ ksoftirqd/0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork_nospec_begin youngest=78s(pid=2), oldest=78s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #-1 __pv_queued_spin_lock_slowpath #0 do_raw_spin_lock #1 _raw_spin_lock_irq #2 __schedule #3 schedule_preempt_disabled #4 cpu_startup_entry #5 start_secondary #6 start_cpu youngest=413s(pid=0), oldest=413s(pid=0) ........................ swapper/3 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule_preempt_disabled #2 cpu_startup_entry #3 rest_init #4 start_kernel #5 x86_64_start_reservations #6 x86_64_start_kernel #7 start_cpu youngest=413s(pid=0), oldest=413s(pid=0) ........................ swapper/0 1 times ** Execution took 1.27s (real) 0.26s (CPU), Child processes: 1.04s