************************ crashinfo ************************* /exports/testreports/38859/testresults/racer-special5-ldiskfs-DNE-centos7_x86_64-centos7_x86_64/oleg341-server-vmcore (3.10.0-7.9-debug) +==========================+ | *** Crashinfo v1.3.7 *** | +==========================+ +++WARNING+++ PARTIAL DUMP with size(vmcore) < 25% size(RAM) KERNEL: /tmp/crash-anaysis.igkbV/vmlinux [TAINTED] DUMPFILE: /exports/testreports/38859/testresults/racer-special5-ldiskfs-DNE-centos7_x86_64-centos7_x86_64/oleg341-server-vmcore [PARTIAL DUMP] CPUS: 4 DATE: Fri Jan 19 12:05:32 EST 2024 UPTIME: 00:11:45 LOAD AVERAGE: 20.70, 8.74, 3.67 TASKS: 428 NODENAME: oleg341-server.virtnet RELEASE: 3.10.0-7.9-debug VERSION: #1 SMP Sat Mar 26 23:28:42 EDT 2022 MACHINE: x86_64 (2399 Mhz) MEMORY: 4 GB PANIC: "Kernel panic - not syncing: LBUG" +--------------------------+ >------------------------| Per-cpu Stacks ('bt -a') |------------------------< +--------------------------+ -- CPU#0 -- PID=15110 CPU=0 CMD=ll_ost00_037 #0 crash_nmi_callback+0x31 #1 nmi_handle+0x97 #2 do_nmi+0x12c #3 end_repeat_nmi+0x1e, 549 bytes of data #4 check_preempt_wakeup+0x0 #5 check_preempt_curr+0x8d #6 ttwu_do_wakeup+0x19 #7 ttwu_do_activate+0x6f #8 try_to_wake_up+0x152 #9 default_wake_function+0x12 #10 autoremove_wake_function+0x18 #11 __wake_up_common+0x70 #12 __wake_up_common_lock+0x83 #13 __wake_up+0x13 #14 ksocknal_queue_tx_locked+0x298 #15 ksocknal_launch_packet+0x19c #16 ksocknal_send+0x14c #17 lnet_ni_send+0x54 #18 lnet_send+0x9a #19 LNetPut+0x2b8 #20 ptl_send_buf+0x140 #21 ptlrpc_send_reply+0x2f3 #22 target_send_reply_msg+0x63 #23 target_send_reply+0x31e #24 tgt_request_handle+0x36c #25 ptlrpc_server_handle_request+0x26e #26 ptlrpc_main+0xc37 #27 kthread+0xe4 #28 ret_from_fork_nospec_begin+0x7 -- CPU#1 -- PID=3074 CPU=1 CMD=socknal_sd00_00 #0 crash_nmi_callback+0x31 #1 nmi_handle+0x97 #2 do_nmi+0x12c #3 end_repeat_nmi+0x1e #-1 string+0x79, 507 bytes of data #4 string+0x79 #5 vsnprintf+0x201 #6 snprintf+0x49 #7 libcfs_idstr+0x6b #8 request_in_callback+0x6d3 #9 ptlrpc_master_callback+0x38 #10 lnet_finalize+0x2e0 #11 ksocknal_process_receive+0x761 #12 ksocknal_scheduler+0x1fa #13 kthread+0xe4 #14 ret_from_fork_nospec_begin+0x7 -- CPU#2 -- PID=14255 CPU=2 CMD=mdt00_021 #0 machine_kexec+0x19e #1 __crash_kexec+0x72 #2 panic+0xf3 #3 lbug_with_loc+0x9b #4 osd_write+0x929 #5 dt_record_write+0x33 #6 llog_osd_write_rec+0xfdb #7 llog_write_rec+0x290 #8 llog_cat_add_rec+0x1d9 #9 llog_add+0x17f #10 osp_sync_add+0x1f9 #11 osp_attr_set+0x3cd #12 lod_sub_attr_set+0x1d7 #13 lod_obj_stripe_attr_set_cb+0x40 #14 lod_obj_for_each_stripe+0x12d #15 lod_attr_set+0x51a #16 lod_layout_change+0x367 #17 dt_layout_change+0x1a140 #18 mdd_layout_change+0x1712 #19 mdt_layout_change+0x2bf #20 mdt_intent_layout+0x910 #21 mdt_intent_opc+0x1c8 #22 mdt_intent_policy+0xfa #23 ldlm_lock_enqueue+0x3b1 #24 ldlm_handle_enqueue+0x359 #25 tgt_enqueue+0x68 #26 tgt_request_handle+0x74e #27 ptlrpc_server_handle_request+0x26e #28 ptlrpc_main+0xc37 #29 kthread+0xe4 #30 ret_from_fork_nospec_begin+0x7 -- CPU#3 -- PID=15133 CPU=3 CMD=ll_ost00_052 #0 crash_nmi_callback+0x31 #1 nmi_handle+0x97 #2 do_nmi+0x12c #3 end_repeat_nmi+0x1e #-1 _raw_spin_unlock_irqrestore+0x17, 507 bytes of data #4 _raw_spin_unlock_irqrestore+0x17 #5 debug_check_no_obj_freed+0x178 #6 kmem_cache_free+0xa0 #7 lnet_md_unlink+0xee #8 __LNetMDUnlink+0x15d #9 ptlrpc_unregister_reply+0x147 #10 ptlrpc_check_set+0x134f #11 ptlrpc_set_wait+0x23b #12 ldlm_run_ast_work+0xde #13 ldlm_handle_conflict_lock+0x72 #14 ldlm_lock_enqueue+0x5c2 #15 ldlm_handle_enqueue+0x359 #16 tgt_enqueue+0x68 #17 tgt_request_handle+0x74e #18 ptlrpc_server_handle_request+0x26e #19 ptlrpc_main+0xc37 #20 kthread+0xe4 #21 ret_from_fork_nospec_begin+0x7 +--------------------------------+ >---------------------| How This Dump Has Been Created |---------------------< +--------------------------------+ *** Panic *** BUG: 73 of 73 active objects replaced +---------------+ >------------------------------| Tasks Summary |------------------------------< +---------------+ Number of Threads That Ran Recently ----------------------------------- last second 241 last 5s 264 last 60s 294 ----- Total Numbers of Threads per State ------ TASK_INTERRUPTIBLE 414 TASK_RUNNING 7 TASK_UNINTERRUPTIBLE 3 TASK_WAKEKILL 1 +++WARNING+++ There are 3 threads running in their own namespaces Use 'taskinfo --ns' to get more details +-----------------------+ >--------------------------| 5 Most Recent Threads |--------------------------< +-----------------------+ PID CMD Age ARGS ----- -------------- ------ ---------------------------- 15133 ll_ost00_052 0 ms (no user stack) 3074 socknal_sd00_00 0 ms (no user stack) 3075 socknal_sd00_01 0 ms (no user stack) 15148 ll_ost00_064 0 ms (no user stack) 15192 ll_ost00_099 0 ms (no user stack) +------------------------+ >-------------------------| Memory Usage (kmem -i) |-------------------------< +------------------------+ PAGES TOTAL PERCENTAGE TOTAL MEM 955067 3.6 GB ---- FREE 624515 2.4 GB 65% of TOTAL MEM USED 330552 1.3 GB 34% of TOTAL MEM SHARED 53869 210.4 MB 5% of TOTAL MEM BUFFERS 51650 201.8 MB 5% of TOTAL MEM CACHED 73703 287.9 MB 7% of TOTAL MEM SLAB 64995 253.9 MB 6% of TOTAL MEM TOTAL HUGE 0 0 ---- HUGE FREE 0 0 0% of TOTAL HUGE TOTAL SWAP 262143 1024 MB ---- SWAP USED 0 0 0% of TOTAL SWAP SWAP FREE 262143 1024 MB 100% of TOTAL SWAP COMMIT LIMIT 739676 2.8 GB ---- COMMITTED 58986 230.4 MB 7% of TOTAL LIMIT +-------------------------------+ >----------------------| Scheduler Runqueues (per CPU) |----------------------< +-------------------------------+ ---+ CPU=0 ---- | CURRENT TASK , CMD=ll_ost00_037 3076 socknal_sd00_02 32.76226 ---+ CPU=1 ---- | CURRENT TASK , CMD=socknal_sd00_00 15169 ll_ost00_077 2.24974 ---+ CPU=2 ---- | CURRENT TASK , CMD=mdt00_021 23 ksoftirqd/2 0.03536 ---+ CPU=3 ---- | CURRENT TASK , CMD=ll_ost00_052 +------------------------+ >-------------------------| Network Status Summary |-------------------------< +------------------------+ TCP Connection Info ------------------- ESTABLISHED 5 LISTEN 3 NAGLE disabled (TCP_NODELAY): 4 user_data set (NFS etc.): 4 UDP Connection Info ------------------- 2 UDP sockets, 0 in ESTABLISHED Unix Connection Info ------------------------ ESTABLISHED 26 CLOSE 17 LISTEN 8 Raw sockets info -------------------- ESTABLISHED 1 Interfaces Info --------------- How long ago (in seconds) interfaces transmitted/received? Name RX TX ---- ---------- --------- lo n/a 703.0 eth0 n/a 0.0 RSS_TOTAL=50972 pages, %mem= 0.8 +------------+ >-------------------------------| Mounted FS |-------------------------------< +------------+ MOUNT SUPERBLK TYPE DEVNAME DIRNAME ffff880138cca000 ffff880139940800 rootfs rootfs / ffff88012a284000 ffff88012a2d0000 sysfs sysfs /sys ffff88012a2841c0 ffff880139944000 proc proc /proc ffff88012a284380 ffff880137668000 devtmpfs devtmpfs /dev ffff88012a284540 ffff8800b523d000 securityfs securityfs /sys/kernel/security ffff88012a284700 ffff88012a2d0800 tmpfs tmpfs /dev/shm ffff88012a2848c0 ffff88013731d000 devpts devpts /dev/pts ffff88012a284a80 ffff88012a2d1000 tmpfs tmpfs /run ffff88012a284c40 ffff88012a2d1800 tmpfs tmpfs /sys/fs/cgroup ffff88012a284e00 ffff88012a2d2000 cgroup cgroup /sys/fs/cgroup/systemd ffff88012a284fc0 ffff88012a2d2800 pstore pstore /sys/fs/pstore ffff88012a285180 ffff88012a2d4800 cgroup cgroup /sys/fs/cgroup/cpuset ffff88012a285340 ffff88012a2d4000 cgroup cgroup /sys/fs/cgroup/net_cls,net_prio ffff88012a285500 ffff88012a2d3800 cgroup cgroup /sys/fs/cgroup/devices ffff88012a2856c0 ffff88012a2d3000 cgroup cgroup /sys/fs/cgroup/perf_event ffff88012a285880 ffff88012a2d5000 cgroup cgroup /sys/fs/cgroup/memory ffff88012a285a40 ffff88012a2d5800 cgroup cgroup /sys/fs/cgroup/cpu,cpuacct ffff88012a285c00 ffff88012a2d6000 cgroup cgroup /sys/fs/cgroup/hugetlb ffff88012a285dc0 ffff88012a2d6800 cgroup cgroup /sys/fs/cgroup/freezer ffff88012a2bc000 ffff88012a2d7000 cgroup cgroup /sys/fs/cgroup/blkio ffff88012a2bc1c0 ffff88012a2d7800 cgroup cgroup /sys/fs/cgroup/pids ffff880137652700 ffff88013766d800 configfs configfs /sys/kernel/config ffff8801376528c0 ffff88013766c800 ext4 /dev/nbd0 / ffff88012a2bc540 ffff8800b5216800 rpc_pipefs rpc_pipefs /var/lib/nfs/rpc_pipefs ffff8801377e6700 ffff8800b420b000 autofs systemd-1 /proc/sys/fs/binfmt_misc ffff880138ccb500 ffff880137668800 mqueue mqueue /dev/mqueue ffff880137652540 ffff880139947800 debugfs debugfs /sys/kernel/debug ffff880137652a80 ffff8800b4dfe800 hugetlbfs hugetlbfs /dev/hugepages ffff880137652c40 ffff8800b4dfa000 binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ ffff88012a2bc700 ffff8800b5212000 ramfs none /mnt ffff8801377e68c0 ffff8800b4296000 squashfs /dev/vda /home/green/git/lustre-release ffff880137652e00 ffff880129de2000 tmpfs none /var/lib/stateless/writable ffff880137652fc0 ffff880129de2000 tmpfs none /var/cache/man ffff880138ccb880 ffff880129de2000 tmpfs none /var/log ffff88012a2bc8c0 ffff880129de2000 tmpfs none /var/lib/dbus ffff880137653180 ffff880129de2000 tmpfs none /tmp ffff880137653340 ffff880129de2000 tmpfs none /var/lib/dhclient ffff880137653500 ffff880129de2000 tmpfs none /var/tmp ffff8801376536c0 ffff880129de2000 tmpfs none /var/lib/NetworkManager ffff880138ccba40 ffff880129de2000 tmpfs none /var/lib/systemd/random-seed ffff8801377e6a80 ffff880129de2000 tmpfs none /var/spool ffff880137653880 ffff880129de2000 tmpfs none /var/lib/nfs ffff88012a2bca80 ffff880129de2000 tmpfs none /var/lib/gssproxy ffff8801377e6c40 ffff880129de2000 tmpfs none /var/lib/logrotate ffff88012a2bcc40 ffff880129de2000 tmpfs none /etc ffff88012a2bce00 ffff880129de2000 tmpfs none /var/lib/rsyslog ffff88012a2bcfc0 ffff880129de2000 tmpfs none /var/lib/dhclient/var/lib/dhclient ffff88012a2bd180 ffff880129d40800 nfs4 192.168.200.253:/exports/state/oleg341-server.virtnet /var/lib/stateless/state ffff88012a2bd880 ffff880129d40800 nfs4 192.168.200.253:/exports/state/oleg341-server.virtnet /boot ffff8801377e6e00 ffff880129d40800 nfs4 192.168.200.253:/exports/state/oleg341-server.virtnet /etc/etc/kdump.conf ffff88012a2bda40 ffff8800b5216800 rpc_pipefs sunrpc /var/lib/nfs/var/lib/nfs/rpc_pipefs ffff8800b1e31500 ffff8800b4296000 squashfs /dev/vda /usr/sbin/mount.lustre ffff8800b1ec4000 ffff8800b420e800 lustre /dev/mapper/mds1_flakey /mnt/lustre-mds1 ffff8800b1ec4e00 ffff88009608b800 lustre /dev/mapper/mds2_flakey /mnt/lustre-mds2 ffff8800b1ec41c0 ffff8800926c4000 lustre /dev/mapper/ost1_flakey /mnt/lustre-ost1 ffff8800b1ec5500 ffff8800827b2800 lustre /dev/mapper/ost2_flakey /mnt/lustre-ost2 +-------------------------------+ >----------------------| Last 40 lines of dmesg buffer |----------------------< +-------------------------------+ [ 706.273791] [] ? _raw_spin_unlock_irq+0xe/0x30 [ 706.275888] [] llog_write_rec+0x290/0x590 [obdclass] [ 706.278814] [] ? __cond_resched+0x26/0x30 [ 706.281103] [] llog_cat_add_rec+0x1d9/0xa50 [obdclass] [ 706.282952] [] ? fld_cache_lookup+0xae/0x1e0 [fld] [ 706.284724] [] llog_add+0x17f/0x1f0 [obdclass] [ 706.286605] [] osp_sync_add+0x1f9/0x760 [osp] [ 706.288046] [] osp_attr_set+0x3cd/0x680 [osp] [ 706.289379] [] ? lod_sub_get_thandle+0x2c7/0x450 [lod] [ 706.290703] [] lod_sub_attr_set+0x1d7/0x500 [lod] [ 706.292583] [] ? osd_attr_set+0x287/0xb00 [osd_ldiskfs] [ 706.294428] [] lod_obj_stripe_attr_set_cb+0x40/0x100 [lod] [ 706.296294] [] lod_obj_for_each_stripe+0x12d/0x310 [lod] [ 706.297724] [] lod_attr_set+0x51a/0xb60 [lod] [ 706.299239] [] ? lod_gen_component_id+0x210/0x210 [lod] [ 706.301140] [] lod_layout_change+0x367/0x3f0 [lod] [ 706.303151] [] ? osd_write_lock+0x5f/0xc0 [osd_ldiskfs] [ 706.304852] [] dt_layout_change+0x20/0xc0 [mdd] [ 706.306227] [] mdd_layout_change+0x1712/0x1db0 [mdd] [ 706.307742] [] mdt_layout_change+0x2bf/0x450 [mdt] [ 706.309254] [] mdt_intent_layout+0x910/0xeb0 [mdt] [ 706.310757] [] mdt_intent_opc+0x1c8/0xc50 [mdt] [ 706.311936] [] ? mdt_intent_open+0x480/0x480 [mdt] [ 706.313290] [] mdt_intent_policy+0xfa/0x460 [mdt] [ 706.315534] [] ldlm_lock_enqueue+0x3b1/0xbb0 [ptlrpc] [ 706.317214] [] ? cfs_hash_rw_unlock+0x15/0x20 [libcfs] [ 706.319334] [] ? cfs_hash_add+0xa6/0x180 [libcfs] [ 706.320736] [] ldlm_handle_enqueue+0x359/0x17c0 [ptlrpc] [ 706.322868] [] ? lustre_msg_buf_v2+0x140/0x1f0 [ptlrpc] [ 706.324567] [] tgt_enqueue+0x68/0x240 [ptlrpc] [ 706.326865] [] tgt_request_handle+0x74e/0x19d0 [ptlrpc] [ 706.331125] [] ptlrpc_server_handle_request+0x26e/0xcf0 [ptlrpc] [ 706.335543] [] ptlrpc_main+0xc37/0x16d0 [ptlrpc] [ 706.337613] [] ? put_prev_entity+0x31/0x400 [ 706.339606] [] ? __switch_to+0xcd/0x4e0 [ 706.340926] [] ? ptlrpc_wait_event+0x5e0/0x5e0 [ptlrpc] [ 706.343049] [] kthread+0xe4/0xf0 [ 706.344034] [] ? kthread_create_on_node+0x140/0x140 [ 706.346863] [] ret_from_fork_nospec_begin+0x7/0x21 [ 706.349391] [] ? kthread_create_on_node+0x140/0x140 ****************************************************************************** ************************ A Summary Of Problems Found ************************* ****************************************************************************** -------------------- A list of all +++WARNING+++ messages -------------------- PARTIAL DUMP with size(vmcore) < 25% size(RAM) There are 3 threads running in their own namespaces Use 'taskinfo --ns' to get more details ------------------------------------------------------------------------------ ** Execution took 12.50s (real) 7.11s (CPU), Child processes: 5.37s